output:

md_document:

variant: markdown_github

setwd("~/HomeWork/Practice")
devtools::install_github("kassambara/datarium")
Skipping install of 'datarium' from a github remote, the SHA1 (f9f9b3a6) has not changed since last install.
  Use `force = TRUE` to force installation
data("marketing", package="datarium")
head(marketing)
data("swiss")
head(swiss)
data("Boston", package="MASS")
head(Boston)

Chapter 3 Regression

Loading Required R packages

library(tidyverse)
library(caret)
theme_set(theme_bw())

Preparing the data

# Load the data
data("marketing", package = "datarium")
# Inspect th data
sample_n(marketing,3)

Split the data into training and test set

set.seed(123)
training.samples <- marketing$sales %>% createDataPartition(p=0.8, list = FALSE)

train.data <- marketing[training.samples,]
test.data <- marketing[-training.samples,]
summary(train.data)
    youtube          facebook       newspaper          sales      
 Min.   :  0.84   Min.   : 0.00   Min.   :  0.36   Min.   : 1.92  
 1st Qu.: 85.95   1st Qu.:11.61   1st Qu.: 19.11   1st Qu.:12.48  
 Median :196.08   Median :25.44   Median : 35.16   Median :15.48  
 Mean   :179.24   Mean   :27.51   Mean   : 39.07   Mean   :16.77  
 3rd Qu.:264.54   3rd Qu.:43.89   3rd Qu.: 55.02   3rd Qu.:20.88  
 Max.   :355.68   Max.   :59.52   Max.   :136.80   Max.   :32.40  
dim(train.data)
[1] 162   4
dim(test.data)
[1] 38  4

Computing linear regression in r

model <- lm(sales ~., data = train.data)
summary(model)

Call:
lm(formula = sales ~ ., data = train.data)

Residuals:
     Min       1Q   Median       3Q      Max 
-10.4122  -1.1101   0.3475   1.4218   3.4987 

Coefficients:
            Estimate Std. Error t value Pr(>|t|)    
(Intercept) 3.391883   0.440622   7.698 1.41e-12 ***
youtube     0.045574   0.001592  28.630  < 2e-16 ***
facebook    0.186941   0.009888  18.905  < 2e-16 ***
newspaper   0.001786   0.006773   0.264    0.792    
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 2.119 on 158 degrees of freedom
Multiple R-squared:  0.8902,    Adjusted R-squared:  0.8881 
F-statistic: 426.9 on 3 and 158 DF,  p-value: < 2.2e-16
# Make predictios
predictions <- model %>% predict(test.data)

# Model Performance
# (a) Prediction error, RMSE
RMSE(predictions, test.data$sales)
[1] 1.590096
# (b) R-square
R2(predictions, test.data$sales)
[1] 0.9380644

Non-Linear Regression

data("Boston", package = "MASS")
set.seed(123)

head(Boston)
training1.samples <- Boston$medv %>% createDataPartition(p=0.8, list = FALSE)

train1.data <- Boston[training.samples,]
test1.data <- Boston[-training.samples, ]

ggplot(train1.data, aes(lstat, medv)) +
  geom_point() +
  stat_smooth()

Let’s start with fitting a linear regression model

model.lm <- lm(medv~lstat, data = train1.data)
summary(model.lm)

Call:
lm(formula = medv ~ lstat, data = train1.data)

Residuals:
   Min     1Q Median     3Q    Max 
-7.889 -3.805 -1.656  2.059 19.972 

Coefficients:
            Estimate Std. Error t value Pr(>|t|)    
(Intercept) 33.98512    0.88151   38.55   <2e-16 ***
lstat       -0.88925    0.06524  -13.63   <2e-16 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 5.548 on 160 degrees of freedom
Multiple R-squared:  0.5373,    Adjusted R-squared:  0.5344 
F-statistic: 185.8 on 1 and 160 DF,  p-value: < 2.2e-16
prediction <- model.lm %>% predict(test1.data)

data.frame(RMSE = RMSE(prediction, test1.data$medv),
           R2 = R2(prediction, test1.data$medv))

Let’s visualize the data

ggplot(train1.data, aes(lstat, medv)) +
  geom_point() +
  stat_smooth(method = lm, formula = y ~ x)

Polynomial regression

medv = b0 + b1 ∗ lstat + b2 ∗ lstat2

mod2 <- lm(medv ~ lstat + I(lstat*lstat), data = train1.data)
summary(mod2)

Call:
lm(formula = medv ~ lstat + I(lstat * lstat), data = train1.data)

Residuals:
    Min      1Q  Median      3Q     Max 
-8.3393 -3.2427 -0.5934  2.0975 16.7311 

Coefficients:
                  Estimate Std. Error t value Pr(>|t|)    
(Intercept)      43.432475   1.274260  34.084  < 2e-16 ***
lstat            -2.520720   0.189206 -13.323  < 2e-16 ***
I(lstat * lstat)  0.053205   0.005921   8.987 7.09e-16 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 4.532 on 159 degrees of freedom
Multiple R-squared:  0.6932,    Adjusted R-squared:  0.6893 
F-statistic: 179.6 on 2 and 159 DF,  p-value: < 2.2e-16
prediction2 <- mod2 %>% predict(test1.data)
data.frame(
  RMSE = RMSE(prediction2, test1.data$medv),
  R2 = R2(prediction2, test1.data$medv)
)
mod22 <- lm(medv ~ poly(lstat,2), data = train1.data)
summary(mod22)

Call:
lm(formula = medv ~ poly(lstat, 2), data = train1.data)

Residuals:
    Min      1Q  Median      3Q     Max 
-8.3393 -3.2427 -0.5934  2.0975 16.7311 

Coefficients:
                Estimate Std. Error t value Pr(>|t|)    
(Intercept)      23.5407     0.3561  66.116  < 2e-16 ***
poly(lstat, 2)1 -75.6187     4.5318 -16.686  < 2e-16 ***
poly(lstat, 2)2  40.7253     4.5318   8.987 7.09e-16 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 4.532 on 159 degrees of freedom
Multiple R-squared:  0.6932,    Adjusted R-squared:  0.6893 
F-statistic: 179.6 on 2 and 159 DF,  p-value: < 2.2e-16
prediction22 <- mod22 %>% predict(test1.data)
data.frame(
  RMSE = RMSE(prediction22, test1.data$medv),
  R2 = R2(prediction22, test1.data$medv)
)
ggplot(train1.data, aes(lstat, medv)) +
  geom_point() +
  stat_smooth(method = lm, formula = y ~ poly(x,2))

ggplot(train1.data, aes(lstat, medv)) +
  geom_point() +
  stat_smooth(method = lm, formula = y ~ (x + I(x*x)))

mod6 <- lm(medv ~ poly(lstat, 6), data = train1.data)
summary(mod6)

Call:
lm(formula = medv ~ poly(lstat, 6), data = train1.data)

Residuals:
     Min       1Q   Median       3Q      Max 
-12.4412  -2.3739  -0.7595   1.7344  15.9725 

Coefficients:
                Estimate Std. Error t value Pr(>|t|)    
(Intercept)      23.5407     0.3304  71.245  < 2e-16 ***
poly(lstat, 6)1 -75.6187     4.2055 -17.981  < 2e-16 ***
poly(lstat, 6)2  40.7253     4.2055   9.684  < 2e-16 ***
poly(lstat, 6)3 -16.9822     4.2055  -4.038 8.45e-05 ***
poly(lstat, 6)4  12.1006     4.2055   2.877  0.00458 ** 
poly(lstat, 6)5  -9.1610     4.2055  -2.178  0.03089 *  
poly(lstat, 6)6  -2.2956     4.2055  -0.546  0.58595    
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 4.206 on 155 degrees of freedom
Multiple R-squared:  0.7424,    Adjusted R-squared:  0.7324 
F-statistic: 74.45 on 6 and 155 DF,  p-value: < 2.2e-16

As we can see in above model summary the polynomial term beyond 5 is not significant

Let’s visualize model

ggplot(train1.data, aes(lstat, medv)) +
  geom_point() +
  stat_smooth(method = lm, formula = y ~ poly(x,6))

mod5 <- lm(medv ~ poly(lstat, 5), data = train1.data)
summary(mod5)

Call:
lm(formula = medv ~ poly(lstat, 5), data = train1.data)

Residuals:
     Min       1Q   Median       3Q      Max 
-13.0370  -2.4259  -0.7941   1.6414  16.1361 

Coefficients:
                Estimate Std. Error t value Pr(>|t|)    
(Intercept)      23.5407     0.3297  71.406  < 2e-16 ***
poly(lstat, 5)1 -75.6187     4.1961 -18.021  < 2e-16 ***
poly(lstat, 5)2  40.7253     4.1961   9.706  < 2e-16 ***
poly(lstat, 5)3 -16.9822     4.1961  -4.047 8.14e-05 ***
poly(lstat, 5)4  12.1006     4.1961   2.884  0.00448 ** 
poly(lstat, 5)5  -9.1610     4.1961  -2.183  0.03051 *  
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 4.196 on 156 degrees of freedom
Multiple R-squared:  0.7419,    Adjusted R-squared:  0.7336 
F-statistic: 89.69 on 5 and 156 DF,  p-value: < 2.2e-16
ggplot(train1.data, aes(lstat, medv)) +
  geom_point() +
  stat_smooth(method = lm, formula = y ~ poly(x,5))

Using log transformation

mod.log <- lm(medv ~ log(lstat), data = train1.data)
summary(mod.log)

Call:
lm(formula = medv ~ log(lstat), data = train1.data)

Residuals:
   Min     1Q Median     3Q    Max 
-7.610 -2.975 -1.048  1.886 17.093 

Coefficients:
            Estimate Std. Error t value Pr(>|t|)    
(Intercept)  50.2130     1.4077   35.67   <2e-16 ***
log(lstat)  -11.5918     0.5928  -19.55   <2e-16 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 4.43 on 160 degrees of freedom
Multiple R-squared:  0.705, Adjusted R-squared:  0.7032 
F-statistic: 382.4 on 1 and 160 DF,  p-value: < 2.2e-16
prediction.log <- mod.log %>% predict(test1.data)

data.frame(
  RMSE = RMSE(prediction.log, test1.data$medv),
  R2 = R2(prediction.log, test1.data$medv)
)
ggplot(data = train1.data, aes(lstat, medv))+
  geom_point() +
  stat_smooth(method = lm, formula = y~log(x))

Spline Regression

library(splines)
knots <- quantile(train1.data$lstat, p = c(0.25, 0.5, 0.75))
model.spline <- lm(medv ~bs(lstat,knots=knots), data = train1.data)
summary(model.spline)

Call:
lm(formula = medv ~ bs(lstat, knots = knots), data = train1.data)

Residuals:
    Min      1Q  Median      3Q     Max 
-9.0823 -2.0512 -0.6894  1.8034 15.8245 

Coefficients:
                          Estimate Std. Error t value Pr(>|t|)    
(Intercept)                 44.032      2.643  16.658  < 2e-16 ***
bs(lstat, knots = knots)1   -1.127      4.160  -0.271    0.787    
bs(lstat, knots = knots)2  -22.744      2.831  -8.034 2.23e-13 ***
bs(lstat, knots = knots)3  -19.776      3.102  -6.376 1.98e-09 ***
bs(lstat, knots = knots)4  -33.442      3.570  -9.366  < 2e-16 ***
bs(lstat, knots = knots)5  -27.995      4.901  -5.713 5.53e-08 ***
bs(lstat, knots = knots)6  -29.583      4.500  -6.574 7.08e-10 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 4.097 on 155 degrees of freedom
Multiple R-squared:  0.7555,    Adjusted R-squared:  0.746 
F-statistic: 79.82 on 6 and 155 DF,  p-value: < 2.2e-16
prediction.spline <- model.spline %>% predict(test1.data)
some 'x' values beyond boundary knots may cause ill-conditioned bases
data.frame(
  RMSE = RMSE(prediction.spline, test1.data$medv),
  R2 = R2(prediction.spline, test1.data$medv)
)
ggplot(train1.data, aes(lstat, medv)) + 
  geom_point() + 
  stat_smooth(method = lm, formula = y~bs(x,df=3))

Generalized additive models

library(mgcv)
Loading required package: nlme

Attaching package: ‘nlme’

The following object is masked from ‘package:dplyr’:

    collapse

This is mgcv 1.8-27. For overview type 'help("mgcv-package")'.
model.additive <- gam(medv ~ s(lstat), data = train1.data)
summary(model.additive)

Family: gaussian 
Link function: identity 

Formula:
medv ~ s(lstat)

Parametric coefficients:
            Estimate Std. Error t value Pr(>|t|)    
(Intercept)  23.5407     0.3261   72.18   <2e-16 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Approximate significance of smooth terms:
           edf Ref.df     F p-value    
s(lstat) 5.825   7.01 65.16  <2e-16 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

R-sq.(adj) =  0.739   Deviance explained = 74.9%
GCV =  17.99  Scale est. = 17.232    n = 162
prediction.additive <- model.additive %>% predict(test1.data)

data.frame(
  RMSE = RMSE(prediction.additive, test1.data$medv),
  R2 = R2(prediction.additive, test1.data$medv)
)
ggplot(train1.data, aes(lstat, medv)) +
  geom_point() +
  stat_smooth(method = gam, formula = y ~ s(x))

Regression Model Accuracy Metrics

Model performance metrics

In regression model, the most commonly known evaluation metrics include:

  1. R-squared (R2), which is the proportion of variation in the outcome that is explained by the predictor variables. In multiple regression models, R2 corresponds to the squared correlation between the observed outcome values and the predicted values by the model. The Higer the R-squared, the better the model.

  2. Root Mean Squared Error (RMSE), which measures the average error performed by the model in the predicting the outcome for an observation. Mathematically, the RMSE is the square root of the mean squared error (MSE), which is the average squared difference between the observed actual outcome values and the values predicted by the model. So, MSE = mean((observeds - predicteds)^2) and RMSE = sqrt(MSE). The lower the RMSE, the better the model.

  3. Residual Standard Error (RSE), also known as the model sigma, is a variant of the RMSE adjusted for the number of predictors in the model. The lower the RSE, the better the model. In practice, the difference between RMSE and RSE is very small, particularly for large multivariate data.

  4. Mean Absolute Error (MAE), like the RMSE, the MAE measures the prediction error. Mathematically, it is the average absolute difference between obsered and predicted out-comes, MAE = mean(abs(observeds - predicteds)) . MAE is less sensitive to outliers compared to RMSE.

The problem with the above metrics, is that they are sensible to the inclusion of additional variables in the model, even if those variables don’t have significant contribution in explaining the outcome. Put in other words, including additional variables in the model will always increase the R2 and reduce the RMSE. So, we need a more robust metric to guide the model choice.

Concerning R2, there is an adjusted version, called Adjusted R-squared, which adjusts the R2 for having too many variables in the model.

Additionally, there are four other important metrics - AIC, AICc, BIC and Mallows Cp - tha are commonly used for model evaluation and selection. These are an unbiased estimate of the model prediction error MSE. The lower these metrics, he better the model.

  1. AIC stands for (Akike’s Information Criteria), a metric developeed by the Japanese Statistician, Hirotugu Akaike, 1970. The basic idea of AIC is to penalize the inclusion of additional variables to a model. It adds a penalty that increases the error when including additional terms. The lowwer the AIC, the better the model.

  2. AICc is a version of AIC corrected for small sample sizes.

  3. BIC (or Bayesian information criteria) is a variant of AIC with a strong penalty for including additional variables to the model.

  4. Mallows Cp: Avariant of AIC developed by Colin Mallows.

Generally, most commonly used metrics, for measuring regression model quality and comparing models, are: Adjusted R2, AIC, BIC and Cp

In the following sections, we’ll sho you how to compute these above mentioned metrics.

library(modelr)
library(broom)

Attaching package: ‘broom’

The following object is masked from ‘package:modelr’:

    bootstrap
# Load the data
data("swiss")

# Inspect the data
sample_n(swiss,3)

Building regression models

We start by creating two models:

  1. Model 1, including all predictors
  2. Model 2, including all predictors except the variable Examination
model.swiss.1 <- lm(Fertility~., data = swiss)
summary(model.swiss.1)

Call:
lm(formula = Fertility ~ ., data = swiss)

Residuals:
     Min       1Q   Median       3Q      Max 
-15.2743  -5.2617   0.5032   4.1198  15.3213 

Coefficients:
                 Estimate Std. Error t value Pr(>|t|)    
(Intercept)      66.91518   10.70604   6.250 1.91e-07 ***
Agriculture      -0.17211    0.07030  -2.448  0.01873 *  
Examination      -0.25801    0.25388  -1.016  0.31546    
Education        -0.87094    0.18303  -4.758 2.43e-05 ***
Catholic          0.10412    0.03526   2.953  0.00519 ** 
Infant.Mortality  1.07705    0.38172   2.822  0.00734 ** 
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 7.165 on 41 degrees of freedom
Multiple R-squared:  0.7067,    Adjusted R-squared:  0.671 
F-statistic: 19.76 on 5 and 41 DF,  p-value: 5.594e-10

data.frame(
  R2 = rsquare(model.swiss.1, data = swiss),
  RMSE = rmse(model.swiss.1, data = swiss),
  MAE = mae(model.swiss.1, data = swiss),
  AIC = AIC(model.swiss.1),
  BIC = BIC(model.swiss.1)
)
model.swiss.2 <- lm(Fertility~. - Examination, data = swiss)

summary(model.swiss.2)

Call:
lm(formula = Fertility ~ . - Examination, data = swiss)

Residuals:
     Min       1Q   Median       3Q      Max 
-14.6765  -6.0522   0.7514   3.1664  16.1422 

Coefficients:
                 Estimate Std. Error t value Pr(>|t|)    
(Intercept)      62.10131    9.60489   6.466 8.49e-08 ***
Agriculture      -0.15462    0.06819  -2.267  0.02857 *  
Education        -0.98026    0.14814  -6.617 5.14e-08 ***
Catholic          0.12467    0.02889   4.315 9.50e-05 ***
Infant.Mortality  1.07844    0.38187   2.824  0.00722 ** 
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 7.168 on 42 degrees of freedom
Multiple R-squared:  0.6993,    Adjusted R-squared:  0.6707 
F-statistic: 24.42 on 4 and 42 DF,  p-value: 1.717e-10
data.frame(
  R2 = rsquare(model.swiss.2, data = swiss),
  RMSE = rmse(model.swiss.2, data = swiss),
  MAE = mae(model.swiss.2, data = swiss),
  AIC = AIC(model.swiss.2),
  BIC = BIC(model.swiss.2)
)
glance(model.swiss.1)
glance(model.swiss.2)

Manual computation of R2, RMSE and MAE

swiss %>% add_predictions(model.swiss.1) %>% summarise(
  R2 = cor(Fertility, pred)^2,
  MSE = mean(Fertility - pred)^2,
  RMSE = sqrt(MSE),
  MAE = mean(abs(Fertility - pred))
)
  • Comparing regression models performance
glance(model.swiss.1) %>% select(adj.r.squared, sigma, AIC, BIC, p.value)
glance(model.swiss.2) %>% select(adj.r.squared, sigma, AIC, BIC, p.value)

From the output above, it can be seen that:

  1. The two models have exactly the samed Adjusted R2 (0.67), meaning that they are equivalent in explaining the outcome, here fertility score. Additionally, they have the same amount of residual standard error (RSE or sigma = 7.17). However, the model 2 is more simple than model 1 because it incorporates less variables. All things equal, the simple model is always better in statistics.

  2. The AIC and the BIC of the model 2 are lower than those of the model1. In model comparison strategies, the model with the lowest AIC and BIC score is preferred.

  3. Finally, the F-statistic p.value of the model 2 is lower than the one of the model 1. This means that the model 2 is statistically more significant compared to model 1, which is consistent to the avove conclusion.

Note that, the RMSE and the RSE are measured in the same scale as the outcome variable. Dividing the RSE by the average value of the outcome variable will give you the prediction error rate, which should be as small as possible:

sigma(model.swiss.1)/mean(swiss$Fertility)
[1] 0.1021544

In our example the average prediction error rate is 10%

Cross-validation

Cross-validation refers to a set of methods for measuring the performance of a given predictive model on new test data sets.

The basic idea, behind cross-validation techniques, consists of dividing the data into two sets:

  1. The training set, used to train (i.e. build) the model;
  2. and the testing set (or validation set), used to test (i.e. validate) the model by estimating the prediction error.

Cross-validation is also known as a resampling method because it involves fitting the same statistical method multiple times using different subsets of the data.

  1. The different cross-validation methods for assessing model performance. We cover the following approches:

    • validation set approach (or data split)
    • Leave On Out Cross Validation
    • k-fold Cross Validation
    • Repeated k-fold Cross Validation

Model performance metrics

After building a model, we are interested in determining the accuracy of this model on predicting the outcome for new unseen observations not used to build the model. Put in other words, we want to estimate the prediction error.

To do so, the basic strategy is to:

  1. Build the model on a training data set
  2. Apply the model on a new test data set to make predictions
  3. Compute the prediction errors

Cross-validation methods

Briefly, cross-validation algorithms can be summarized as follow:

  1. Reserve a small sample of the data set
  2. Build (or train) the model using the remaining part of the data set
  3. Test the effectiveness of the model on the reserved sample of the data set. If the model works well on the test data set, then it’s good.

The following sections describe the diffferent cross-validation techniques.

1. The Validation set Approach

The validation set approach consists of randomly splitting the data into two sets: one set is used to train the model and the remaining other set is used to test the model.

The process works as follow:

  1. Build (train) the model on the training data set
  2. Appply the model to the test data set to predict the outcome of new unseen observations
  3. Quantify the prediction error as the mean squared difference between the observed and the predicted outcome values.

The example below splits the swiss data set so that 80% is used for training a linear regression model and 20% is used to evaluate the model performance.

# Split the data into training and test set 
set.seed(123)

training.samples <- swiss$Fertility %>%
  createDataPartition(p=0.8, list = FALSE)

train.swiss.data <- swiss[training.samples, ]
test.swiss.data <- swiss[-training.samples, ]

# Build the model

model.swiss.lm <- lm(Fertility ~ ., data = train.swiss.data)

# Make predictions and compute the R2, RMSE and MAE
predictions.swiss.lm <- model.swiss.lm %>%  predict(test.swiss.data)

data.frame(
  R2 = R2(predictions.swiss.lm, test.swiss.data$Fertility),
  RMSE = RMSE(predictions.swiss.lm, test.swiss.data$Fertility),
  MAE = MAE(predictions.swiss.lm, test.swiss.data$Fertility)
)

When comparing two models, the one that produces the lowest test sample RMSE is the preferred model.

The RMSE and the MAE are measured in the same scale as the outcome variable. Dividing the RMSE by the average value of the outcome variable will give you the prediction error rate, which sould be as small as possible:

RMSE(predictions.swiss.lm, test.swiss.data$Fertility)/mean(test.swiss.data$Fertility)
[1] 0.1281514

Note that, the validation set method is only useful when you have a large data set that can be partitioned. A disadvantage is that we build a model on a fraction of the data set only, possibly leaving out some interesting information about data, leading to higher bias. Therefore, the test error rate can be highly variable, depending on which observations are included in the training set and which observation are included in the validation set.

2. Leave one out cross validation - LOOCV

This method works as follows: 1. Leave out one data point and build the model on the rest of the data set 2. Test the model against the data point that is left out at step 1 and record the test error associated with the prediction 3. Repeat the process for all data points 4. Compoute the overall prediction error by taking the average of all these test error estimates recoreded at step3.

Practical example in R using the caret package:

# Define training control
train.control <- trainControl(method = "LOOCV")

# Train the model

model.loocv <- train(Fertility ~., data = swiss, method = "lm", trControl = train.control)
# Summarize the results
print(model.loocv)
Linear Regression 

47 samples
 5 predictor

No pre-processing
Resampling: Leave-One-Out Cross-Validation 
Summary of sample sizes: 46, 46, 46, 46, 46, 46, ... 
Resampling results:

  RMSE      Rsquared   MAE     
  7.738618  0.6128307  6.116021

Tuning parameter 'intercept' was held constant at a value of TRUE

The advantage of the LOOCV method is that we make use all data points reducing potential bias.

However, the process is repeated as many times as there are data points, resulting to a higher exection time when n is extreamely large.

Additionally, we test the model performance against one data, point at each iteration. This might result to higher variation in the prediction error, if some data points are outliers. So, we need a good ratio of testing data points, a solution provided by the k-fold cross-validation method.

K-fold cross-validation

The k-fold cross validation method evaluates the model performance on different subset of the training data and then calculate the average prediction error rate. The algorithm is as follow:

  1. Randomly split the data into k-subset (or k-fold) (for example 5 subsets)
  2. Reserve on subset and train the model on all other subsets
  3. Test the model on the reserved subset and record the prediction error
  4. Repeat this procss untill each of the k subsets has served as the test set.
  5. Compute the average of the k recorded errors. This is called the cross validation error serving as the performance metric for the model

K-fold cross-validation (CV) is a robust method for estimating the accuracy of a model. The most obvious advantage of k-fold CV compared to LOOCV is computational. A less obvious but potentially more important advantage of k-fold CV is that it often fives more accurate estimates of the test error rate than does LOOCV

In practice, one typically performs k-fold cross-validation using k = 5 or k = 10, as these values have been shown empirically to yield test error rate estimates that suffer neither from excessively high bias nor from very high variance.

The following example uses 10-fold cross validation to estimate the prediction error. Make sure to set seed for reproducibility.

# Define training control
set.seed(123)

train.control <- trainControl(method = "cv", number = 10)

# Train the model
model.cv <- train(Fertility~., data = swiss, method = "lm", trControl = train.control)

# Summarize the results
print(model.cv)
Linear Regression 

47 samples
 5 predictor

No pre-processing
Resampling: Cross-Validated (10 fold) 
Summary of sample sizes: 43, 42, 42, 41, 43, 41, ... 
Resampling results:

  RMSE      Rsquared   MAE     
  7.379432  0.7512317  6.032731

Tuning parameter 'intercept' was held constant at a value of TRUE

4. Repeated K-fold cross-validation

The process of splitting the data into k-folds can be repeated a number of times, this is called repeated k-fold cross validation.

Bootstrap procedure

The bootstrap method is used to quantify the uncertainty with a given statistical estimator or with a predictive model.

It consists of randomly selecting a sample of n observations from the original data set. This subset, called bootstrap data set is then used to evaluate the model.

This procedure is repeated a large number of times and the standard error of the bootstrap estimate is then calculated. The results provide an indication of the variance of the model performance.

Note that, the sampling is performed with replacement, which means that the same observation can occur more than once in the boostrrap data set.

Evaluating a predictive model performance

# Define training control

train.control.bootstrap <- trainControl(method = "boot", number = 100)

model.bootstrap <- train(Fertility ~., data = swiss, method = "lm", trControl = train.control.bootstrap)

# Summarize the results
print(model.bootstrap)
Linear Regression 

47 samples
 5 predictor

No pre-processing
Resampling: Bootstrapped (100 reps) 
Summary of sample sizes: 47, 47, 47, 47, 47, 47, ... 
Resampling results:

  RMSE      Rsquared   MAE     
  8.432357  0.6048287  6.791066

Tuning parameter 'intercept' was held constant at a value of TRUE

Quantifying an estimator uncertainty and confidence intervals

The bootstrap approach can be used to quantify the uncertainty (or standard error) associated with any given statistical estimator.

For example, you might want to estimate the accuracy of the linear regression beta coefficients using bootstrap method.

The different steps are as follows:

  1. Create a simple function, model_coef(), that takes the swiss data set as well as the indices for the observations, and returns the regression coefficients.
  2. Apply the function boot_fun() to the full data set of 47 observations in order to compute the coefficients

We start by creating a function that retrns the regression model coefficeients:

model_coef <- function(data, index) {
  coef(lm(Fertility~., data = data, subset = index))
}

model_coef(swiss, 1:47)
     (Intercept)      Agriculture      Examination        Education         Catholic Infant.Mortality 
      66.9151817       -0.1721140       -0.2580082       -0.8709401        0.1041153        1.0770481 
library(boot)

Attaching package: ‘boot’

The following object is masked from ‘package:lattice’:

    melanoma
boot(swiss, model_coef, 500)

ORDINARY NONPARAMETRIC BOOTSTRAP


Call:
boot(data = swiss, statistic = model_coef, R = 500)


Bootstrap Statistics :
      original       bias    std. error
t1* 66.9151817  0.572138874 10.84264891
t2* -0.1721140 -0.004654338  0.06456320
t3* -0.2580082 -0.027889510  0.25405701
t4* -0.8709401  0.002319556  0.21736743
t5*  0.1041153 -0.002166432  0.03067023
t6*  1.0770481  0.008581232  0.43205390

In the output above,

  • original column corresponds to the regression coefficients. The associated standard errors are given in the column std.error .

  • t1 corresponds to the intercept, t2 conrresponds to *Agriculture** and so on ..

For example, it can be observe that, the standard error (SE) of the regression coefficient associated with Agriculture is 0.06

Note that, the standard errors measure the variability/accuracy of the beta coefficients. It can be used to compute the confidence intervals of the coefficients.

For example, the 95% confidence interval for a given coefficient b is defined as b+/-1.96*SE(b), where:

  • The lower limits of b = b-1.96*SE(b) = -0.172 - (1.96*0.0680) = -0.306(for Agriculture variable)
  • The Upper limits of b = b+1.96*SE(b) = -0.172 + (1.96*0.0680) = -0.03743121(for Agriculture variable)

That is, there is approximately a 95% chance that the interval[-0.306,-0.036] will contain the true value of the coefficient.

Using the standard lm() function gives a slightly different standard errors, because the linear model make some assumptions about the data:

summary(lm(Fertility ~., data = swiss))$coef
                   Estimate  Std. Error   t value     Pr(>|t|)
(Intercept)      66.9151817 10.70603759  6.250229 1.906051e-07
Agriculture      -0.1721140  0.07030392 -2.448142 1.872715e-02
Examination      -0.2580082  0.25387820 -1.016268 3.154617e-01
Education        -0.8709401  0.18302860 -4.758492 2.430605e-05
Catholic          0.1041153  0.03525785  2.952969 5.190079e-03
Infant.Mortality  1.0770481  0.38171965  2.821568 7.335715e-03

The bootstrap approach does not rely on any of these assumptions made by the linear model and so it is likely giving a more accurate estimate on the coefficients standard errors than tis the summary() function.

Model Selection

Best Subsets Regression

library(tidyverse)
library(caret)
library(leaps)

Computing best subset regression

The R function regsubsets() [leaps package] can be used to identify different best models of different sizes.

You need to specify the option nvmax, which represents the maximum number of predictors to incorporate in the mode. For example, if nvmax = 5, the function will return up to the best 5-variables model, that is, it returns the best 1-varable model, the best 2-variables model, .., the best 5-variables models.

In our example, we have only 5 predictor variables in the data. So, we’ll use nvmax = 5

models.regsubsets <- regsubsets(Fertility~., data= swiss, nvmax = 5)
summary(models.regsubsets)
Subset selection object
Call: regsubsets.formula(Fertility ~ ., data = swiss, nvmax = 5)
5 Variables  (and intercept)
                 Forced in Forced out
Agriculture          FALSE      FALSE
Examination          FALSE      FALSE
Education            FALSE      FALSE
Catholic             FALSE      FALSE
Infant.Mortality     FALSE      FALSE
1 subsets of each size up to 5
Selection Algorithm: exhaustive
         Agriculture Examination Education Catholic Infant.Mortality
1  ( 1 ) " "         " "         "*"       " "      " "             
2  ( 1 ) " "         " "         "*"       "*"      " "             
3  ( 1 ) " "         " "         "*"       "*"      "*"             
4  ( 1 ) "*"         " "         "*"       "*"      "*"             
5  ( 1 ) "*"         "*"         "*"       "*"      "*"             

Model selection criteria: Adjusted R2, Cp and BIC

The summary() function returns some metrics - Adjusted R2, Cp and BIC allowing us to identify the best overall model, where best is defined as the model that maximize the adjusted R2 and minimize the prediction errors

The adjusted R2 represents the proportion of variation, in the outcome, that are explained by variation in predictors values. The higher the adjusted R2, the better the model.

The best model, according to each of these metrics, can be extracted as follow:

res.sum <- summary(models.regsubsets)

data.frame(
  Adj.R2 = which.max(res.sum$adjr2),
  CP = which.min(res.sum$cp),
  BIC = which.min(res.sum$bic)
)
NA

There is no single correct solution to model selection, each of these criteria will lead to slightly different models.

Remembert that,

“all models are wrong, some models are useful”

So, we have different “best” models depending on which metrics we consider. We need additonal strategies.

Note also that the adjusted R2, BIC and Cp are calculated on the training data that have been used to fit the model. This means that, the model selection, using these metrics, is possibly subject to overfitting and may not perform as well when applied to new data.

A more rigorous approach is to select a models based on the prediction error computed on a new test data usin k-fold cross-validation techniques

K-fold cross-validation

Here, we’ll follow the procedure below:

  1. Extract the different model formulas from the models object
  2. Train a linear model on the formula using k-fold cross-validation(with k=5) and compute the prediction error of each model

We start by defining two helper function:

  1. get_model_formula(), allowing to access easily the formula of the models returned by the function regsubsets().
#id: model id
#object: regsubsets object
#data: data used to fit regsubsets

get_model_formula <- function(id, object){
  models <- summary(object)$which[id,-1]
  #get outcome variable
  
  form <- as.formula(object$call[[2]])
  outcome <- all.vars(form)[1]
  #Get model predictors
  predictors <- names(which(models == TRUE))
  predictors <- paste(predictors, collapse = "+")
  # Build model formula
  as.formula(paste(outcome, "~", predictors))
}
get_model_formula(3,models.regsubsets)
Fertility ~ Education + Catholic + Infant.Mortality
<environment: 0x7f9c38bcc868>
  1. get_cv_error(), to get the cross-validation (CV) error for a given model:
get_cv_error <- function (model.formula, data){
  set.seed(1)
   train.control <- trainControl(method = "cv", number = 5)
   cv <- train(model.formula, data = data, method = "lm", trControl = train.control)
   cv$results$RMSE
}

Use the above defined method to compute the prediction error

# Compute cross-validation error

model.ids <- 1:5
cv.errors <- map(model.ids, get_model_formula, models.regsubsets) %>%
  map(get_cv_error, data = swiss) %>%
  unlist()
cv.errors
[1] 9.422610 8.452344 7.926889 7.678508 7.923860
# Select the model that minimize the CV error
which.min(cv.errors)
[1] 4

It can be seen that the model with 4 variables is the best model. It has the lower prediction error. The regression coefficients of this model can be extracted as follow:

coef(models.regsubsets, 4)
     (Intercept)      Agriculture        Education         Catholic Infant.Mortality 
      62.1013116       -0.1546175       -0.9802638        0.1246664        1.0784422 

Stepwise Regression

  1. Forward selection: Which starts with no predictors in the model, Iteratively adds the most contributive predictor, and stops when the improvement is no longer statistically significant.

  2. Backwoard selection(or backward elimination): which starts with all predictors in the model (full model), iteratively removes the least contributive predictors, and stops when you have a model where all predictors are statistically significant.

  3. Stepwise selection (or sequential replacement) which is a combination of forward and backward selections. you start with no predictors, then sequentially add the most contributive predictors (like forward selection). After adding each new variable, remove any variables that no longer provide and improvement in the model fit

library(tidyverse)
library(caret)
library(leaps)

Computing setpwise regression

library(MASS)

Attaching package: ‘MASS’

The following object is masked from ‘package:dplyr’:

    select
# Fit the full model
full.model <- lm(Fertility ~., data = swiss)

# Stepwise regression model
step.model <- stepAIC(full.model, direction = "both", trace = FALSE)

summary(step.model)

Call:
lm(formula = Fertility ~ Agriculture + Education + Catholic + 
    Infant.Mortality, data = swiss)

Residuals:
     Min       1Q   Median       3Q      Max 
-14.6765  -6.0522   0.7514   3.1664  16.1422 

Coefficients:
                 Estimate Std. Error t value Pr(>|t|)    
(Intercept)      62.10131    9.60489   6.466 8.49e-08 ***
Agriculture      -0.15462    0.06819  -2.267  0.02857 *  
Education        -0.98026    0.14814  -6.617 5.14e-08 ***
Catholic          0.12467    0.02889   4.315 9.50e-05 ***
Infant.Mortality  1.07844    0.38187   2.824  0.00722 ** 
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 7.168 on 42 degrees of freedom
Multiple R-squared:  0.6993,    Adjusted R-squared:  0.6707 
F-statistic: 24.42 on 4 and 42 DF,  p-value: 1.717e-10

Penalized Regression: Ridge, Lasso and Elastic Net

Shrinkage methods

Ridge regression

Ridge regression shrinks the regression coefficients, so that varibles, with minor contribution to the outcome, have their coefficients close to zero.

The shrinkage of the coefficients is achieved by penalizing the regression model with a penalty term called L2-norm, which is the sum of the squared coefficients.

The amount of the penalty can be fine-tuned using a constant called lambda . Selecting a good value for lambda is critical

When lambda = 0, the penalty term has no effect, and ridge regression will produce the calssical least square coefficients. However, as lambda increases to infinite, the impact of the shrinkage penalty grows, and the ridge regression coefficients will get close zero.

Note that, in contrast to the ordinary least square regression, ridge regression is highly affected by the scale of the predictors. Therfore, it is better to standardize (i.e., scale) the predictors before applying the ridge regression, so that all the predictors are on the same scale.

The standardization of a predictor x, can be achieved using the formula x` = x/sd(x), where sd(x) is the stadard deviation of . The consequence of this is that, all standardized predictors will have a standard deviation of one allowing the final fit to not depend on the scale on which the predictors are measured.

One important advantage of the ridge regression, is that it still perfroms will, compared to the ordinary least square method, in a situation where you have a large multivariate data with the number of predictors (p) larger than the number of observations (n).

One disadvantage of the ridge regression is that, it will included all the predictors in the final model, unlike the stepwise regression methods, which will genrally select models that involve a reduced set of variables.

Ridge regression shrinks the coefficients towards zero, but it will not set any of them exactly to zero. The lasso regression is an alternative that overcomes this drawback.

Lasso regression

Lasso stands for Least Absolute Shrinkage and Selection Operator. It shrinks the regression coefficients toward zero by penalizing the regression model with a penalty term called L1-norm, which is the sum of the absolute coefficients.

In the case of lasso regression, the penalty has the effect of forcing some of the coefficient estimates, with a minor contribution to the model, to be ecactly equal to zero. This means that, lasso can be also seen as an alternative to the subset selection methods for performing variable selection in order to reduce the complexity of the model.

As in ridge regression, selecting a good value of lambda for the lasso is critical.

One obvious advantage of lasso regression over ridge regression, is that it produces simpler and more interpretable models that incorporate only a reduced set of the predictors. However, neither ridge regression nor the lasso will universally dominate the orther.

Generally, lasso mght perform better in a situation where some of the predictors have large coefficients, and the remaining predictors have very small coefficients.

Ridge regression will perform better when the outcome is a function of many predictors, all with coefficents of roughly equal size.

Cross-validation methods can be used for identifying which of these two techniques is better on a particular data set.

Elastic Net

Elastic Net produces a regression model that is penalized with both the L1-norm and L2-norm. The consequence of this is to effectively shrink coefficents (Like in ridge regression) and to set some coefficients to zero (as in LASSO).

Loading required R packages

library(tidyverse)
library(caret)
library(glmnet)
Loading required package: Matrix

Attaching package: ‘Matrix’

The following objects are masked from ‘package:tidyr’:

    expand, pack, unpack

Loading required package: foreach

Attaching package: ‘foreach’

The following objects are masked from ‘package:purrr’:

    accumulate, when

Loaded glmnet 2.0-18

Preparing the data

We’ll use the Boston data set [in MASS package], for predicting the median house value(mdev), in Boston Suburbs, based on multiple predictor variables.

# Load the data
data("Boston", package = "MASS")

# Split the data into training and test set
set.seed(123)
boston.samples <- Boston$medv %>%
  createDataPartition(p=0.8, list = FALSE)

train.boston.data <- Boston[boston.samples, ]
test.boston.data <- Boston[-boston.samples, ]

Computing penalized linear regression

You need to create two objects:

  • y for storing the outcome variable
  • x for holding the predictor variables.
x <- model.matrix(medv~., train.boston.data)[,-1]
# Outcome variable
y <- train.boston.data$medv

We’ll use the R function glmnet() fro computing penalized linear regression models.

glmnet(x,y, alpha = 1, lambda = NULL)

Call:  glmnet(x = x, y = y, alpha = 1, lambda = NULL) 

      Df    %Dev   Lambda
 [1,]  0 0.00000 6.908000
 [2,]  1 0.09343 6.294000
 [3,]  2 0.18670 5.735000
 [4,]  2 0.26620 5.225000
 [5,]  2 0.33220 4.761000
 [6,]  2 0.38700 4.338000
 [7,]  2 0.43240 3.953000
 [8,]  2 0.47020 3.602000
 [9,]  2 0.50150 3.282000
[10,]  3 0.53390 2.990000
[11,]  3 0.56100 2.725000
[12,]  3 0.58350 2.482000
[13,]  3 0.60210 2.262000
[14,]  3 0.61770 2.061000
[15,]  3 0.63050 1.878000
[16,]  3 0.64120 1.711000
[17,]  3 0.65010 1.559000
[18,]  3 0.65740 1.421000
[19,]  3 0.66360 1.294000
[20,]  4 0.66990 1.179000
[21,]  4 0.67560 1.075000
[22,]  4 0.68030 0.979100
[23,]  4 0.68420 0.892200
[24,]  4 0.68750 0.812900
[25,]  5 0.69080 0.740700
[26,]  5 0.69380 0.674900
[27,]  6 0.69730 0.614900
[28,]  7 0.70200 0.560300
[29,]  7 0.70610 0.510500
[30,]  8 0.70960 0.465200
[31,]  8 0.71430 0.423800
[32,]  8 0.71820 0.386200
[33,]  8 0.72140 0.351900
[34,] 10 0.72520 0.320600
[35,] 11 0.72850 0.292100
[36,] 11 0.73120 0.266200
[37,] 11 0.73350 0.242500
[38,] 11 0.73540 0.221000
[39,] 11 0.73690 0.201400
[40,] 11 0.73820 0.183500
[41,] 12 0.73950 0.167200
[42,] 12 0.74220 0.152300
[43,] 12 0.74440 0.138800
[44,] 12 0.74620 0.126500
[45,] 12 0.74760 0.115200
[46,] 12 0.74880 0.105000
[47,] 12 0.74990 0.095660
[48,] 12 0.75070 0.087160
[49,] 12 0.75140 0.079420
[50,] 12 0.75200 0.072370
[51,] 12 0.75250 0.065940
[52,] 12 0.75290 0.060080
[53,] 12 0.75320 0.054740
[54,] 12 0.75350 0.049880
[55,] 12 0.75370 0.045450
[56,] 12 0.75390 0.041410
[57,] 12 0.75410 0.037730
[58,] 12 0.75420 0.034380
[59,] 12 0.75430 0.031330
[60,] 12 0.75440 0.028540
[61,] 12 0.75450 0.026010
[62,] 12 0.75460 0.023700
[63,] 12 0.75460 0.021590
[64,] 12 0.75460 0.019670
[65,] 12 0.75470 0.017930
[66,] 12 0.75470 0.016330
[67,] 12 0.75470 0.014880
[68,] 12 0.75480 0.013560
[69,] 12 0.75480 0.012360
[70,] 12 0.75480 0.011260
[71,] 12 0.75480 0.010260
[72,] 12 0.75480 0.009346
[73,] 12 0.75480 0.008516
[74,] 12 0.75480 0.007760
  • x: matrix of predictor variables
  • y: the response or outcome variable, which is a binary variable.
  • alpha: the elasticnet mixing parameter. Allowed values include:
    • “1”: for lasso regression
    • “0”: for ridge regression
    • a valued between 0 and 1 (say 0.25) for elastic net regression.
    • lambda: a numeric value defining the amount of shrinkage. Should be specify by analyst

In penalized regression, you need to specify a constant lambda to adjust the amount of the coefficent shrinkage. The best lambda for your data, can be defined as the lambda that minimize the cross-validation prediction error rate. This can be determined automatically using the function cv.glmnet().

In the following section, we start by computing ridge, lasso and elastic net regresion models. Next, we’ll compare the different models in order to choose the best one for our data The best model is defined as the model that has the lowest prediction error, RMSE

Computing ridge regression

# Find the best lambda using cross-validation
set.seed(123)
cv <- cv.glmnet(x,y, alpha = 0)

# Display the best lambda value
cv$lambda.min
[1] 0.6907672
# Fit the final model on the training data
model.ridge <- glmnet(x,y, alpha = 0, lambda = cv$lambda.min)

# Display regression coefficients
coef(model.ridge)
14 x 1 sparse Matrix of class "dgCMatrix"
                       s0
(Intercept)  29.172942853
crim         -0.073961766
zn            0.035006472
indus        -0.055702961
chas          2.485565613
nox         -11.449222575
rm            3.976849313
age          -0.002944308
dis          -1.221350102
rad           0.147920742
tax          -0.006358355
ptratio      -0.869860292
black         0.009399874
lstat        -0.483051940
# Make predictions on the test data
x.test <- model.matrix(medv~., test.boston.data)[,-1]
predictions.ridge <- model.ridge %>% predict(x.test) %>% as.vector()

# Model performance metrics

data.frame(
  RMSE = RMSE(predictions.ridge, test.boston.data$medv),
  Rsquare = R2(predictions.ridge, test.boston.data$medv)
)

Note that by default, the function glmnet() standardizes variables so that their scales are comparable. However, the coefficients are always returned on the original scale.

Computing lasso regression

The only difference between the R code used for ridge regression is that for lasso regression you need to specify the argument alpha = 1 instead of alpha - 0

# Find the best lambda using cross-validation
set.seed(123)
cv <- cv.glmnet(x,y,alpha = 1)
# Display the best lambda value
cv$lambda.min
[1] 0.007759554
# Fit the final model on the training data
model <- glmnet(x,y, alpha = 1, lambda = cv$lambda.min)
#Display regression coefficients
coef(model)
14 x 1 sparse Matrix of class "dgCMatrix"
                       s0
(Intercept)  36.962374270
crim         -0.092549948
zn            0.048543171
indus        -0.008321076
chas          2.287418592
nox         -16.832483690
rm            3.810180182
age           .          
dis          -1.598716155
rad           0.286797839
tax          -0.012456750
ptratio      -0.950997301
black         0.009652895
lstat        -0.528739166
# Make predictions on the test data
x.laso.test <- model.matrix(medv ~., test.boston.data)[,-1]
predictions.lasso <- model %>% predict(x.test) %>% as.vector()

# Model performance metrics
data.frame(
  RMSE = RMSE(predictions.lasso, test.boston.data$medv),
  Rsquare = R2(predictions.lasso, test.boston.data$medv)
)

Computing elastic net regression

The elastic net regression can be easily computed using the caret workflow, which invokes the glmnet package.

We use caret to automatically select the best tuning parameters alpha and lambda. the caret packages tests a range of possible alpha and lambda values, then selects the best values fro lambda and alpha, resulting to a final modl that is an elastic net modle.

Here, we’ll test the combination of 10 different values for alpha and lambda. This is specified using the option tuneLength.

The best alpha and lambda values are those values that minimize the cross-validation error

# # Build the model using the training set
# set.seed(123)
# model.elastic <- train(
#   medv ~., data = train.boston.data, method = "glmnet",
#   trcontrol = trainControl("cv", number = 5),
#   tuneLength = 10
# )
# 
# # Best tuning parameter
# model.elastic$bestTune

Principal Component and Partial Least Squares Regression

Principal component regression

The principal component regression(PCR) first applies Principal Component Analysis on the data set to summarize the original predictor variables into few new variables also known as principal component (PCs), which are a linear combination of the original data.

These PCs are then used to build the linear regression model. The number of principal components, to incorporate in the model, is chosen by corss-validation (cv). Note that, PCR is suitable when the data set contains highly correlated predictors.

Partial least squares regression

A possible drawback of PCR is that we have no guarantee that the selected principal components are associated with the outcome. Here, the selection of the principal components to incorporate in the model is not supervised by the outcome variable.

An alternative to PCR is the Partial Least Squares (PLS) regression, which identifies new principal components that not only summarizes the original predictors, but also that are related to the outcome. These components are then used to fit the regression model. So compared PCR, PLS uses a dimension reduction strategy that is supervised by the outcome.

Like PCR, PLS is convenient for data with highly-correlated predictors. The number of PCs used in PLS is generally chosen by cross-validation. Predictors and the outcome variables should be generally standardized, to make the variables comparable.

Loading required R packages

  • tidyverse for easy data manipulation and visualization
  • caret for easy machine learning workflow
  • pls, for computing PCR and PLS
library(tidyverse)
library(caret)
library(pls)

Attaching package: ‘pls’

The following object is masked from ‘package:caret’:

    R2

The following object is masked from ‘package:stats’:

    loadings

Preparing the data

We’ll use the boston data set

Computing principal component regression

# Build the model on training set

set.seed(123)

model.pc <- train(
  medv~., data = train.boston.data, method = "pcr",
  scale = TRUE,
  trControl = trainControl("cv", number = 10),
  tuneLength = 10
)

# Plot model RMSE vs different values of components

plot(model.pc)


#Print the best tunning parameter ncomp that 
# minimize the cross-validation error, RMSE
model.pc$bestTune
summary(model.pc$finalModel)
Data:   X dimension: 407 13 
    Y dimension: 407 1
Fit method: svdpc
Number of components considered: 5
TRAINING: % variance explained
          1 comps  2 comps  3 comps  4 comps  5 comps
X           47.48    58.40    68.00    74.75    80.94
.outcome    38.10    51.02    64.43    65.24    71.17
# Make predictions
predictions.pc <- model.pc %>% predict(test.boston.data)
# Model performance metrics

data.frame(
  RMSE = caret::RMSE(predictions.pc, test.boston.data$medv),
  Rsquare = caret::R2(predictions.pc, test.boston.data$medv)
)

The plot shows the prediction error made by the model acccording to the number of principal components incorporated in the model.

Our analysis shows that, choosing five principal components (ncomp = 5) gives the smallest prediction error RMSE.

The summary() function also provides the percentage of variance explained in the predictors(x) and in the outcome (medv) using different numbers of components.

For example, 80.94% of the variation (or information) contained in the predictors are captured by 5 components (ncomp = 5). Additionally, setting ncomp = 5, captures 71% of the information in the outcome variable (medv), which is good

Taken together, cross-validation identifies ncomp - 5 as the optimal number of PCs that minimize the prediction error (RMSE) and explains enough variation in the predictors and in the outcome

Computing partial least squares

# Build the model on training set
set.seed(123)
model.pls <- train(
  medv~., data = train.boston.data, method = "pls",
  scale = TRUE,
  trControl = trainControl("cv", number = 10),
  tuneLength = 10
)

# Plot model RMSE vs Different values of components
plot(model.pls)

model.pls$bestTune
summary(model.pls$finalModel)
Data:   X dimension: 407 13 
    Y dimension: 407 1
Fit method: oscorespls
Number of components considered: 9
TRAINING: % variance explained
          1 comps  2 comps  3 comps  4 comps  5 comps  6 comps  7 comps  8 comps  9 comps
X           46.19    57.32    64.15    69.76    75.63    78.66    82.85    85.92    90.36
.outcome    50.90    71.84    73.71    74.71    75.18    75.35    75.42    75.48    75.49
#Make predictions
predictions.pls <- model.pls %>% predict(test.boston.data)

# Model performance metrics

data.frame(
  RMSE = caret::RMSE(predictions.pls, test.boston.data$medv),
  Rsquare = caret::R2(predictions.pls, test.boston.data$medv)
)

The optimal number of principal components included in the PLS model is 9. This captures 90% of the variation in the predictors and 75% of the variation in the outcome variable (medv).

In our example, the cross-validation error RMSE obtained with the PLS model is lower than the RMSE obtained using the PCR method. So, the PLS model is the best model, for explaining our data, compaed to the PCR model.

Classification

In this part, we’ll comver the follwing topics:

Most of the classification algorithms computes the probability of belonging to a given class.

Observations are then assigned to the class that have the highest probability score.

Generaly, you need to decide a probability cutoff above which you consider the an observation as belonging to a given class.

Dataset will be using

  1. PimaIndiansDiabetes2 data set

The Pima Indian Diabetes data set is available in the mlbench package. It will be used for binary classification.

# Load the data set
data("PimaIndiansDiabetes2", package = "mlbench")

# Inspect the data
head(PimaIndiansDiabetes2,4)
str(PimaIndiansDiabetes2)
'data.frame':   768 obs. of  9 variables:
 $ pregnant: num  6 1 8 1 0 5 3 10 2 8 ...
 $ glucose : num  148 85 183 89 137 116 78 115 197 125 ...
 $ pressure: num  72 66 64 66 40 74 50 NA 70 96 ...
 $ triceps : num  35 29 NA 23 35 NA 32 NA 45 NA ...
 $ insulin : num  NA NA NA 94 168 NA 88 NA 543 NA ...
 $ mass    : num  33.6 26.6 23.3 28.1 43.1 25.6 31 35.3 30.5 NA ...
 $ pedigree: num  0.627 0.351 0.672 0.167 2.288 ...
 $ age     : num  50 31 32 21 33 30 26 29 53 54 ...
 $ diabetes: Factor w/ 2 levels "neg","pos": 2 1 2 1 2 1 2 1 2 2 ...

The data contains 768 individuals (female) and 9 clinical variables for predictin the probability of individuals in being diabete-positive or negative:

Column Names Description
pregnant Number of times pregnant
glucose Plasma glucose concentration (glucose tolerance test)
pressure Diastolic blood pressure (mm Hg)
triceps Triceps skin fold thickness (mm)
insulin 2-Hour serum insulin (mu U/ml)
mass Body mass index (weight in kg/(height in m)^2)
pedigree Diabetes pedigree function
age Age (years)
diabetes Class variable (test for diabetes)

Performing the following steps might improve the accuracy of your model

set.seed(123)

training.Pima.samples <- PimaIndiansDiabetes2$diabetes %>%
  createDataPartition(p = 0.8, list = FALSE)

train.pima.data <- PimaIndiansDiabetes2[training.Pima.samples, ]
test.pima.data <- PimaIndiansDiabetes2[-training.Pima.samples, ]

Computing logistic regression

# Fit the model
model.lr.pima <- glm(diabetes~., data = train.pima.data, family = binomial)

# Summarize the model
summary(model.lr.pima)

Call:
glm(formula = diabetes ~ ., family = binomial, data = train.pima.data)

Deviance Residuals: 
    Min       1Q   Median       3Q      Max  
-2.6751  -0.6666  -0.3588   0.6581   2.5650  

Coefficients:
              Estimate Std. Error z value Pr(>|z|)    
(Intercept) -10.196902   1.369247  -7.447 9.54e-14 ***
pregnant      0.082032   0.061219   1.340  0.18025    
glucose       0.036517   0.006381   5.723 1.05e-08 ***
pressure      0.001333   0.013053   0.102  0.91863    
triceps       0.008425   0.018649   0.452  0.65145    
insulin      -0.001219   0.001441  -0.845  0.39784    
mass          0.081434   0.030448   2.675  0.00748 ** 
pedigree      1.186528   0.470790   2.520  0.01173 *  
age           0.030886   0.020337   1.519  0.12884    
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 402.38  on 314  degrees of freedom
Residual deviance: 280.83  on 306  degrees of freedom
  (300 observations deleted due to missingness)
AIC: 298.83

Number of Fisher Scoring iterations: 5
## Make predictions

probabilities <- model.lr.pima %>% predict(test.pima.data, type = "response")
predicted.classes <- ifelse(probabilities > 0.5, "pos", "neg") 
# predicted.classes
# Model accuracy
mean(predicted.classes == test.pima.data$diabetes)
[1] NA

Simple logistic regression

The simple logistic regression is used to predict the probability of class membership based on one single predictor variable.

The following R code builds a model to predict the probability of being diabetes-positive based on the plasma glucose concentration:

model.logit.simple <- glm(diabetes ~ glucose, data = train.pima.data, family = binomial)
summary(model.logit.simple)

Call:
glm(formula = diabetes ~ glucose, family = binomial, data = train.pima.data)

Deviance Residuals: 
    Min       1Q   Median       3Q      Max  
-2.1322  -0.7882  -0.5157   0.8641   2.2706  

Coefficients:
             Estimate Std. Error z value Pr(>|z|)    
(Intercept) -5.555585   0.482107  -11.52   <2e-16 ***
glucose      0.039188   0.003697   10.60   <2e-16 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 790.12  on 610  degrees of freedom
Residual deviance: 637.56  on 609  degrees of freedom
  (4 observations deleted due to missingness)
AIC: 641.56

Number of Fisher Scoring iterations: 4

The output above shows the estimate of the regression beta coefficients and their significance levels. The intercept (b0) is -5.55 and the coefficient of glucose variable is 0.039.

The logistic equation can be written as p = exp(-5.55 + 0.039*glucose)/[1+exp(-5.55+0.039*glucose)]. Using this formula, for each new glucose plasma concentration value, you can predict the probability of the individuals in bein diabetes positive.

Predictions can be easily made using the function predict(). Use the option type = “response” to directly obtain the probabilities

newdata <- data.frame(glucose = c(20,190))
probabilities <- model.logit.simple  %>% predict(newdata, type = "response")
predicted.classes <- ifelse(probabilities > 0.5, "pos", "neg")
predicted.classes
    1     2 
"neg" "pos" 
train.pima.data %>%
  mutate(prob = ifelse(diabetes == "pos", 1, 0)) %>%
  ggplot(aes(glucose,prob)) +
  geom_point(alpha = 0.2) +
  geom_smooth(method = "glm", method.args = list(family = "binomial"))+
  labs(
    title = "Simple Logistic Regression Model",
    x = "Plasma Glucose Concentration",
    y = "Probability of being diabetes-pos"
  )

plot(model.logit.simple)

Stepwise Logistic Regression

Stepwise logistic regression consists of automatically selecting a reduced number of predictor variables for building the best performing logistic regression model.

Data set: PimaIndiansDiabetes2

Quick start R code

library(MASS)
# Fit the model

removed.missing.data <- na.omit(train.pima.data)
model <- glm(diabetes ~ ., data = removed.missing.data , family = binomial) %>% stepAIC(trace = FALSE)

summary(model)

Call:
glm(formula = diabetes ~ glucose + mass + pedigree + age, family = binomial, 
    data = removed.missing.data)

Deviance Residuals: 
    Min       1Q   Median       3Q      Max  
-2.8024  -0.6631  -0.3716   0.6617   2.5631  

Coefficients:
              Estimate Std. Error z value Pr(>|z|)    
(Intercept) -10.081719   1.202909  -8.381  < 2e-16 ***
glucose       0.033770   0.005536   6.101 1.06e-09 ***
mass          0.083808   0.022726   3.688 0.000226 ***
pedigree      1.110791   0.456960   2.431 0.015064 *  
age           0.051063   0.014602   3.497 0.000471 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 402.38  on 314  degrees of freedom
Residual deviance: 283.63  on 310  degrees of freedom
AIC: 293.63

Number of Fisher Scoring iterations: 5
removed.missing.data.from.test <- na.omit(test.pima.data)

probabilities.step <- model %>% predict(removed.missing.data.from.test, type = "response")
predicted.classes.step.logit <- ifelse(probabilities.step > 0.5, "pos", "neg")

#Model Accuracy
mean(predicted.classes.step.logit == removed.missing.data.from.test$diabetes)
[1] 0.7922078
df <- data.frame("Predicted" = predicted.classes.step.logit)
(df$Predicted == test.pima.data$diabetes)
longer object length is not a multiple of shorter object lengthlonger object length is not a multiple of shorter object length
  [1]  TRUE FALSE  TRUE  TRUE FALSE  TRUE  TRUE  TRUE FALSE  TRUE FALSE FALSE FALSE  TRUE  TRUE FALSE  TRUE  TRUE  TRUE  TRUE FALSE
 [22]  TRUE  TRUE FALSE  TRUE FALSE  TRUE  TRUE  TRUE  TRUE  TRUE  TRUE  TRUE FALSE  TRUE  TRUE  TRUE  TRUE  TRUE FALSE FALSE  TRUE
 [43] FALSE FALSE FALSE  TRUE FALSE  TRUE FALSE  TRUE  TRUE FALSE  TRUE FALSE FALSE  TRUE FALSE  TRUE FALSE FALSE FALSE FALSE  TRUE
 [64]  TRUE FALSE  TRUE  TRUE  TRUE  TRUE FALSE  TRUE FALSE  TRUE  TRUE FALSE  TRUE  TRUE  TRUE FALSE FALSE  TRUE  TRUE  TRUE FALSE
 [85] FALSE  TRUE  TRUE FALSE  TRUE  TRUE  TRUE  TRUE  TRUE  TRUE  TRUE  TRUE FALSE  TRUE  TRUE  TRUE FALSE  TRUE  TRUE  TRUE  TRUE
[106] FALSE  TRUE FALSE  TRUE FALSE  TRUE FALSE  TRUE FALSE FALSE  TRUE FALSE  TRUE FALSE  TRUE  TRUE FALSE FALSE FALSE FALSE FALSE
[127]  TRUE  TRUE FALSE  TRUE FALSE  TRUE FALSE FALSE  TRUE FALSE FALSE  TRUE  TRUE  TRUE  TRUE  TRUE  TRUE FALSE  TRUE  TRUE FALSE
[148] FALSE  TRUE FALSE  TRUE FALSE  TRUE

Penalized logistic regression

Penalized logistic regression imposes a penalty to the logistic model for having too many variables. This results in shrinking the coefficients of the less contributive variables toward zero. This is also know as regularization. The most commonly used penalized regression include

  • Ridge regression: variables with minor contribution have their coefficients close to zero. However, all the variables are incorporated in the model. This is useful when all variables need to be incorporated in the model according to domain knowledge.

  • lasso regression: the coefficients of some less contributive variables aer forced to be exactly zero. Only the most significant variables ae kept in the final model.

  • elastic net regression: the combination of ridge and lasso regression. It shrinks some coefficients toward zero (like ridge regerssion) and set some coefficients to exactly zero (like lasso regression)

Required packages

library(tidyverse)
library(caret)
library(glmnet)

Preparing the data

Data set: PimaIndiansDiabetes2

# Load the data and remove NAs
Pima.Indians.data <- na.omit(PimaIndiansDiabetes2)
# Inspect the data
sample_n(Pima.Indians.data, 3)

# Split the data into training and test set
set.seed(123)

training.samples <- Pima.Indians.data$diabetes %>%
  createDataPartition(p = 0.8, list = FALSE)

train.pima.indians <- Pima.Indians.data[training.samples, ]
test.pima.indians <- Pima.Indians.data[-training.samples, ]

Additional data preparation

The R function model.matrix() help to create the matrix of predictors and also automatically converts categorical predictors to appropriate dummy variables, which is required for the glmnet() function.

# Dummy code categorical predictor variables
x <- model.matrix(diabetes ~., train.pima.indians)[,-1]

# Convert the outcome (class) to a numerical variable
y <- ifelse(train.pima.indians$diabetes == "pos", 1, 0)

We’ll use the R function glmnet() for computing penalized logistic regression.

The simplified format is as follow:

library(glmnet)
glmnet(x, y, family = "binomial", alpha = 1, lambda = NULL)

Call:  glmnet(x = x, y = y, family = "binomial", alpha = 1, lambda = NULL) 

      Df      %Dev    Lambda
 [1,]  0 3.147e-15 0.2480000
 [2,]  1 3.690e-02 0.2259000
 [3,]  1 6.743e-02 0.2059000
 [4,]  1 9.292e-02 0.1876000
 [5,]  1 1.144e-01 0.1709000
 [6,]  1 1.325e-01 0.1557000
 [7,]  1 1.479e-01 0.1419000
 [8,]  1 1.610e-01 0.1293000
 [9,]  1 1.722e-01 0.1178000
[10,]  2 1.851e-01 0.1073000
[11,]  2 1.970e-01 0.0978000
[12,]  2 2.071e-01 0.0891100
[13,]  2 2.158e-01 0.0812000
[14,]  2 2.233e-01 0.0739800
[15,]  3 2.314e-01 0.0674100
[16,]  5 2.400e-01 0.0614200
[17,]  5 2.484e-01 0.0559600
[18,]  5 2.558e-01 0.0509900
[19,]  5 2.620e-01 0.0464600
[20,]  5 2.674e-01 0.0423400
[21,]  5 2.721e-01 0.0385700
[22,]  6 2.762e-01 0.0351500
[23,]  6 2.798e-01 0.0320300
[24,]  6 2.829e-01 0.0291800
[25,]  6 2.856e-01 0.0265900
[26,]  6 2.879e-01 0.0242300
[27,]  6 2.898e-01 0.0220700
[28,]  6 2.915e-01 0.0201100
[29,]  6 2.929e-01 0.0183300
[30,]  6 2.941e-01 0.0167000
[31,]  6 2.951e-01 0.0152100
[32,]  6 2.959e-01 0.0138600
[33,]  6 2.967e-01 0.0126300
[34,]  7 2.975e-01 0.0115100
[35,]  7 2.984e-01 0.0104900
[36,]  7 2.993e-01 0.0095550
[37,]  7 2.999e-01 0.0087060
[38,]  7 3.005e-01 0.0079330
[39,]  7 3.010e-01 0.0072280
[40,]  7 3.014e-01 0.0065860
[41,]  7 3.018e-01 0.0060010
[42,]  8 3.022e-01 0.0054680
[43,]  8 3.025e-01 0.0049820
[44,]  8 3.028e-01 0.0045390
[45,]  8 3.030e-01 0.0041360
[46,]  8 3.032e-01 0.0037690
[47,]  8 3.034e-01 0.0034340
[48,]  8 3.035e-01 0.0031290
[49,]  8 3.037e-01 0.0028510
[50,]  8 3.038e-01 0.0025980
[51,]  8 3.038e-01 0.0023670
[52,]  8 3.039e-01 0.0021570
[53,]  8 3.040e-01 0.0019650
[54,]  8 3.040e-01 0.0017900
[55,]  8 3.040e-01 0.0016310
[56,]  8 3.041e-01 0.0014860
[57,]  8 3.041e-01 0.0013540
[58,]  8 3.041e-01 0.0012340
[59,]  8 3.042e-01 0.0011240
[60,]  8 3.042e-01 0.0010250
[61,]  8 3.042e-01 0.0009336
[62,]  8 3.042e-01 0.0008506
[63,]  8 3.042e-01 0.0007750
set.seed(123)
cv.lasso <- cv.glmnet(x,y, alpha = 1, family = "binomial")
# cv.lasso
# summary(cv.lasso)
# Fit the final model on the training data
model.logit.lasso <- glmnet(x,y, alpha = 1, family = "binomial", lambda = cv.lasso$lambda.min)

# Display regression coefficients
coef(model.logit.lasso)
9 x 1 sparse Matrix of class "dgCMatrix"
                       s0
(Intercept) -8.6158632778
pregnant     0.0350263984
glucose      0.0369158677
pressure     .           
triceps      0.0164757684
insulin     -0.0003928031
mass         0.0304940618
pedigree     0.7854910473
age          0.0362782651
# Make predictions on the test data
x.test <- model.matrix(diabetes~., test.pima.indians)[,-1]
probabilities.logit.lasso <- model.logit.lasso %>% predict(newx = x.test)
predict.classess.logit.lasso <- ifelse(probabilities.logit.lasso > 0.5, "pos", "neg")

#Model accuracy
observed.classes <- test.pima.indians$diabetes
mean(predict.classess.logit.lasso == observed.classes)
[1] 0.7692308
plot(cv.lasso)

The plot displays the cross-validation error according to the log of lambda. The left dashed vertical line indicates that the log of the optimal value of lambda is approximately -5, which is the one that minimizes the prediction error. This lambda value will give the most accurate model. The exact value of lambda can be viewed as follow:

cv.lasso$lambda.min
[1] 0.008706319
cv.lasso$lambda.1se
[1] 0.06740987

Using lambda.min as the best lambda, gives the following regression coefficients:

coef(cv.lasso, cv.lasso$lambda.min)
9 x 1 sparse Matrix of class "dgCMatrix"
                        1
(Intercept) -8.6156146833
pregnant     0.0350758913
glucose      0.0369156360
pressure     .           
triceps      0.0164842605
insulin     -0.0003924262
mass         0.0304848446
pedigree     0.7855063693
age          0.0362650459
coef(cv.lasso, cv.lasso$lambda.1se)
9 x 1 sparse Matrix of class "dgCMatrix"
                       1
(Intercept) -4.657500725
pregnant     .          
glucose      0.026284471
pressure     .          
triceps      0.001905497
insulin      .          
mass         .          
pedigree     .          
age          0.017338402

Logistic Regression Assumptions and Diagnostics

The logistic regression method assumes that:

  • The outcome is a binary or dichotomous variable like yes vs no, positive vs negative, 1 vs 0.

  • there is alineare relationship between the logit of the outcome and each predictor variables. Recall that the outcome and each predictor variables. Recall that the logit function is logit(p) = log(p/(1-p)), where p is the probabilities of the outcome

  • There is no influential value (extreame values or outliers) in the continuous predictors

  • There is no high intercorrelation (i.e. multicollinearity) among the predictors.

Loading reequired R packages

library(tidyverse)
library(broom)
theme_set(theme_classic())
PimaIndiansDiabetes2.cleaned <- na.omit(PimaIndiansDiabetes2)
# Fit the logistic regression model
model.pima.logit2 <- glm(diabetes~., data = PimaIndiansDiabetes2.cleaned, family = binomial)
# Predict the probability (p) of diiabete positivity
probabilities.pima.logit <- predict(model.pima.logit2, type = "response")
predicted.pima.classes <- ifelse(probabilities.pima.logit > 0.5, "pos", "neg")
head(predicted.pima.classes)
    4     5     7     9    14    15 
"neg" "pos" "neg" "pos" "pos" "pos" 
# train.pima.indians
mydata <- PimaIndiansDiabetes2.cleaned  %>%
  dplyr::select_if(is.numeric)
predictors <- colnames(mydata)
# Bind the logit and tidying the data for plot
model.pima.logit <- mydata %>%
  mutate(logit = log(probabilities.pima.logit/(1-probabilities.pima.logit))) %>%
  gather(key = "predictors", value = "predictor.value", -logit)
ggplot(model.pima.logit, aes(logit, predictor.value)) +
  geom_point(size = 0.5, alpha = 0.5) +
  geom_smooth(method = "loess") +
  theme_bw() +
  facet_wrap(~predictors, scales = "free_y")

Influential values

Influential values are extreme individual data points that can alter the quaity of the logistic regression model. The most extreme values in the data can be examined by visualizing the Cook’s distance values.

Here we label the top 3 largest values:

plot(model.pima.logit2, which = 4, id.n = 3)

Note that, not all outliers are influential observations. To check kwhether the data contains potential influential observations, the standardized residual error can be inspected. Data points with an absolute standardized residuals above 3 represent possible outliers and may deserve closer attention.

The following R code computes the standardized residuals (.std.resid) and the Cook’s distance (.cooksd) using the R function augment()[broom package].

# Extract model results

model.pima.logit.data <- augment(model.pima.logit2) %>%
  mutate(index = 1:n())

The data for the top 3 largest values, according to the Cook’s distance, can be displayed as follow:

model.pima.logit.data %>% top_n(3, .cooksd)
NA

Plot the standardized residuals:

ggplot(model.pima.logit.data, aes(index, .std.resid)) +
  geom_point(aes(color = diabetes), alpha = 0.5) +
  theme_bw()

Filter potential influential data points with abs(.std.res) > 3

model.pima.logit.data %>% filter(abs(.std.resid) > 3)

There is no influential observation in our data.

When we have outliers in a continuous predictor, potential solutions include:

  • Removing the concerned records
  • Transform the data into log scale
  • Use non parameteric methods

Multicollinearity

Multicollinearity corresponds to a situation where the data contain highly correlated predictor variables.

Multicollinearity is an important issue in regression analysis and should be fixed by removing the concerned variables. It can be assessed using the R function vif()

car::vif(model.pima.logit2)
pregnant  glucose pressure  triceps  insulin     mass pedigree      age 
1.892387 1.378937 1.191287 1.638865 1.384038 1.832416 1.031715 1.974053 

As a rule of thumb, a VIF value that exceeds 5 or 10 indicates a problematic amount of collinearity. In our example there is no collinearity: all variables have a value of VIF well below 5.

Multinomial logistic regression

The multinomial logistic regressin is an extension of the logistic regression for multclass classifcation tasks. It is used when the outcome involves more than two classes.

Loading required R packages

library(tidyverse)
library(caret)
library(nnet)

Attaching package: ‘nnet’

The following object is masked from ‘package:mgcv’:

    multinom

Preparing the data

We’ll use the iris data set

# Load the data
data("iris")
# Inspect the data
sample_n(iris,3)

# Split the data into training and test set
set.seed(123)
training.samples.iris <- iris$Species %>% createDataPartition(p=0.8, list = FALSE)
train.iris.data <- iris[training.samples.iris,]
test.iris.data <- iris[-training.samples.iris,]

Computing multinomial logistic regression

# Fit the model
model <- nnet::multinom(Species ~., data = train.iris.data)
# weights:  18 (10 variable)
initial  value 131.833475 
iter  10 value 13.707358
iter  20 value 5.548255
iter  30 value 5.196272
iter  40 value 4.985881
iter  50 value 4.978698
iter  60 value 4.970034
iter  70 value 4.968625
iter  80 value 4.967145
iter  90 value 4.967125
iter 100 value 4.967097
final  value 4.967097 
stopped after 100 iterations
# Summarize the model
summary(model)
Call:
nnet::multinom(formula = Species ~ ., data = train.iris.data)

Coefficients:
           (Intercept) Sepal.Length Sepal.Width Petal.Length Petal.Width
versicolor    17.29267    -5.643165   -8.391525     14.61331   -2.091772
virginica    -21.99694    -6.650591  -14.538985     22.01169   14.158731

Std. Errors:
           (Intercept) Sepal.Length Sepal.Width Petal.Length Petal.Width
versicolor    43.11205     119.7240    198.5332     91.66177    59.26249
virginica     43.68199     119.7252    198.5908     91.81816    59.59031

Residual Deviance: 9.934194 
AIC: 29.93419 
# Make predictions

predicted.species <- model %>% predict(test.iris.data)
head(predicted.species)
[1] setosa setosa setosa setosa setosa setosa
Levels: setosa versicolor virginica
# Model accuracy
mean(predicted.species == test.iris.data$Species)
[1] 0.9666667

Discriminant Analysis

The following discriminant analysis methods will be described:

  • Linear discriminant analysis (LDA): Uses linear combinations of predictors to predict the class of a given observation. Assumes that the predictor variables (p) are normally distributed and the classes have identical variances (for univariate analysis, p = 1) or identical covariance matrices (for multvariate analysis, p > 1).

  • Quadratic discriminant analysis (QDA): More flexible than LDA. Here, there is no assumption that the covariance matrix of classes is the same

  • Mixture discriminant analysis (MDA): Each class is assumed to be a Gaussian mixture of subclasses.

  • Flexible Discriminant Analysis (FDA): Non-linear combination of predictors is used such as splines.

  • Regularized discriminant analysis (RDA): Regularization (or shrinkage) improves the estimate of the covariance matrices in situation where the number of predictors is larger than the number of samples in the training data. This leads to an improvement of the discriminant analysis.

Loading required R packages

library(tidyverse)
library(caret)
theme_set(theme_classic())

Preparing the data

  1. Split the data
# Lodd the data
data("iris")
# Split the data into training and test set
set.seed(123)
training.samples.iris <- iris$Species %>%
  createDataPartition(p=0.8, list = FALSE)

train.data.iris <- iris[training.samples.iris, ]
test.data.iris <- iris[-training.samples.iris, ]
  1. Normalize the data. Categorical
# Estimate preprocessing parameters
preproc.pram <- train.data.iris %>%
  preProcess(method = c("center", "scale"))

# Transform the data using the estimated parameters
train.transformed <- preproc.pram %>% predict(train.data.iris)
test.transformed <- preproc.pram %>% predict(test.data.iris)

Before performing LDA, consider: * Inspecting the univariate distribution of each variable and make sure that they are normally distribute. If not, you can transform them using log and root for exponential distributions and Box-Cox for skewed distributions.

*removing outliers from your data and standardize the variables to make their scale comparable.

R code

library(MASS)

# Fit the model
model.lda <- lda(Species ~., data = train.transformed)
# Make predictions
predictions <- model.lda %>% predict(test.transformed)

# Model accuracy
mean(predictions$class == test.transformed$Species)
[1] 1
model.lda
Call:
lda(Species ~ ., data = train.transformed)

Prior probabilities of groups:
    setosa versicolor  virginica 
 0.3333333  0.3333333  0.3333333 

Group means:
           Sepal.Length Sepal.Width Petal.Length Petal.Width
setosa       -1.0120728   0.7867793   -1.2927218  -1.2496079
versicolor    0.1174121  -0.6478157    0.2724253   0.1541511
virginica     0.8946607  -0.1389636    1.0202965   1.0954568

Coefficients of linear discriminants:
                    LD1         LD2
Sepal.Length  0.9108023  0.03183011
Sepal.Width   0.6477657  0.89852536
Petal.Length -4.0816032 -2.22724052
Petal.Width  -2.3128276  2.65441936

Proportion of trace:
   LD1    LD2 
0.9905 0.0095 
summary(model.lda)
        Length Class  Mode     
prior    3     -none- numeric  
counts   3     -none- numeric  
means   12     -none- numeric  
scaling  8     -none- numeric  
lev      3     -none- character
svd      2     -none- numeric  
N        1     -none- numeric  
call     3     -none- call     
terms    3     terms  call     
xlevels  0     -none- list     

LDA determines group means and computes, for each individual, the probability of belonging to the different groups. The individual is then affected to the group with the highest probability score.

The lda() outputs contain the following elements: * prior probabilities of groups: the proportion of training observation in each group. For example, there are 33% of the training observations in the setosa group * Group means: group center of gravity. Shows the mean of each variable in each group. * Coefficients of linear discriminants: Shows the linear combination of predictor variables that are used to form the LDA decisionrule. For example

LD1 = 0.01*Sepal.Length + 0.64*Sepal.Width - 4.08*Petal.Length = 2.3*Petal.Width LD2 = 0.03*Sepal.Length + 0.89*Sepal.Width - 2.2*Petal.Length - 2.6*Petal.Width

Using the function Plot() produces plots of the linear discriminants, obtained by computing LD1 and LD2 for each of the training observations

plot(model.lda)

lda.data <- cbind(train.transformed, predict(model.lda)$x)

ggplot(lda.data, aes(LD1, LD2)) +
  geom_point(aes(color = Species))

It can be seen that, our model correctly classified 100% of observations, which is excellent.

Note that, by default, the probability cutoff used to decide group-membership is 0.5

In some situation, you might want to increase the precision of the model. In this case you can fine-tune the model by adjusting the posterior probability cutoff. For example, you can increase or lower the cutoff

Variable selection:

Note that, if the predictor variables are standardized before computing LDA, the discriminator weights can be used as measures of variable importance for feature selection

Quardratic discriminant analysis - QDA

QDA is little bit more flexible than LDA, in the sense that it does not assumes the equality of variance/covariance. In other words, for QDA the covariance matrix can be different for each class.

LDA tends to be a better than QDA then you have a small training set.

In contrast, QDA is recommended if the training set is very large, so that the variance of the classifier is not a major issue, or if the assumption of a common covariance matrix for the K classes if clearly untenable

QDA can be computed using the R function qda() [MASS package]

library(MASS)
#Fit the model
model.qda <- qda(Species~., data = train.transformed)
model.qda
Call:
qda(Species ~ ., data = train.transformed)

Prior probabilities of groups:
    setosa versicolor  virginica 
 0.3333333  0.3333333  0.3333333 

Group means:
           Sepal.Length Sepal.Width Petal.Length Petal.Width
setosa       -1.0120728   0.7867793   -1.2927218  -1.2496079
versicolor    0.1174121  -0.6478157    0.2724253   0.1541511
virginica     0.8946607  -0.1389636    1.0202965   1.0954568
# Make predictions
predictions.qda <- model.qda %>% predict(test.transformed)

# Model accuracy
mean(predictions.qda$class == test.transformed$Species)
[1] 1

Mixture discriminant analysis

The LDA classifier assumes that each class comes from a single normal (or Gaussian ) distribution.

This is too restrictive.

For MDA, ther are classes, and each class is assumed to be a Gaussian mixture of subclasses, where each data point has a probability of belonging to each class. Equality of covariance matrix, among classes, is still assumed

library(mda)
Loading required package: class
Loaded mda 0.4-10
model.mda <- mda(Species~., data = train.transformed)
model.mda
Call:
mda(formula = Species ~ ., data = train.transformed)

Dimension: 5 

Percent Between-Group Variance Explained:
    v1     v2     v3     v4     v5 
 96.77  99.63  99.86 100.00 100.00 

Degrees of Freedom (per dimension): 5 

Training Misclassification Error: 0.025 ( N = 120 )

Deviance: 14.005 
# Make predictions

predicted.classes.mda <- model.mda %>% predict(test.transformed)

# Model accuracy
mean(predicted.classes.mda == test.transformed$Species)
[1] 1

Flexible discriminant analysis - FDA

FDA is a flexible extension of LDA that uses non-linear combinations of predictors such as splines. FDA is useful to model multivariate non-normality or non-linear relationships among variables within each group, alowing for a more accurate classification

library(mda)
# Fit the model
model.fda <- fda(Species~., data = train.transformed)
model.fda
Call:
fda(formula = Species ~ ., data = train.transformed)

Dimension: 2 

Percent Between-Group Variance Explained:
    v1     v2 
 99.05 100.00 

Degrees of Freedom (per dimension): 5 

Training Misclassification Error: 0.025 ( N = 120 )
predicted.classes.fda <- model.fda %>% predict(test.transformed)
#Model accuracy
mean(predicted.classes.fda == test.transformed$Species)
[1] 1

Regularized discriminant analysis

RDA builds a classification rule by regularizing the group covariance matrices allowing a more robust model against multicollinearity in the data. This might be very useful for a large multivariate data set containing highly correlated predictors.

Regularized discriminant analysis is a kind of a trade-off between LDA and QDA. Recall that, in LDA we assume equality of covariance matrix for all of the classes. QDA assumes different covariance matrices for all the classes. QDa assumes different covariance matrices for all the classes. Regularized discriminant analysis is an intermediate between LDA and QDA.

RDA shrinks the separte covariances of QDA toward a common covariance as in LDA. This improves the estimate of the covariance matrices in situations where the number of predictors is larger thant the number of samples in the training data, potentially leading to an improvement of the model accuracy.

library(klaR)
# Fit the model
model.rda <- rda(Species~., data = train.transformed)

#  Make predictions
predictions.rda <- model.rda %>% predict(test.transformed)

# Model accuracy

mean(predictions.rda$class == test.transformed$Species)
[1] 1

Naive Bayes Classifier

The Naive Bayes Classifier is a simple and powerful method that can be used for binary and multiclass classification problems.

Naive Bayes classifier predicts the class membership probability of observations using Bayes theorem, which is based on conditional probability, that is the probability of something to happen, given that something else has already occured.

Will use PimaIndiansDiabetes2 data set

Computing Naive Bayes

library(klaR)
# Fit the model
model.naive <- NaiveBayes(diabetes ~., data = train.pima.indians)

model.naive
$apriori
grouping
      neg       pos 
0.6687898 0.3312102 

$tables
$tables$pregnant
        [,1]     [,2]
neg 2.890476 2.645282
pos 4.471154 3.968215

$tables$glucose
        [,1]     [,2]
neg 112.1381 25.23393
pos 147.1635 29.39590

$tables$pressure
        [,1]     [,2]
neg 69.19048 12.13173
pos 73.54808 13.71264

$tables$triceps
        [,1]      [,2]
neg 27.83333 10.668822
pos 32.68269  9.539152

$tables$insulin
        [,1]     [,2]
neg 137.0571 110.5755
pos 214.2115 140.5018

$tables$mass
        [,1]     [,2]
neg 32.11524 6.905892
pos 35.42500 6.825278

$tables$pedigree
         [,1]      [,2]
neg 0.4699952 0.3091545
pos 0.6139904 0.3853125

$tables$age
       [,1]      [,2]
neg 28.6619  9.085715
pos 35.8750 10.248076


$levels
[1] "neg" "pos"

$call
NaiveBayes.default(x = X, grouping = Y)

$x
    pregnant glucose pressure triceps insulin mass pedigree age
4          1      89       66      23      94 28.1    0.167  21
5          0     137       40      35     168 43.1    2.288  33
7          3      78       50      32      88 31.0    0.248  26
9          2     197       70      45     543 30.5    0.158  53
14         1     189       60      23     846 30.1    0.398  59
15         5     166       72      19     175 25.8    0.587  51
17         0     118       84      47     230 45.8    0.551  31
19         1     103       30      38      83 43.3    0.183  33
20         1     115       70      30      96 34.6    0.529  32
26        10     125       70      26     115 31.1    0.205  41
33         3      88       58      11      54 24.8    0.267  22
44         9     171      110      24     240 45.4    0.721  54
51         1     103       80      11      82 19.4    0.491  22
52         1     101       50      15      36 24.2    0.526  26
53         5      88       66      21      23 24.4    0.342  30
54         8     176       90      34     300 33.7    0.467  58
55         7     150       66      42     342 34.7    0.718  42
57         7     187       68      39     304 37.7    0.254  41
58         0     100       88      60     110 46.8    0.962  31
64         2     141       58      34     128 25.4    0.699  24
69         1      95       66      13      38 19.6    0.334  25
70         4     146       85      27     100 28.9    0.189  27
71         2     100       66      20      90 32.9    0.867  28
74         4     129       86      20     270 35.1    0.231  23
83         7      83       78      26      71 29.3    0.767  36
86         2     110       74      29     125 32.4    0.698  27
88         2     100       68      25      71 38.5    0.324  26
89        15     136       70      32     110 37.1    0.153  43
92         4     123       80      15     176 32.0    0.443  34
93         7      81       78      40      48 46.7    0.261  42
95         2     142       82      18      64 24.7    0.761  21
96         6     144       72      27     228 33.9    0.255  40
98         1      71       48      18      76 20.4    0.323  22
99         6      93       50      30      64 28.7    0.356  23
100        1     122       90      51     220 49.7    0.325  31
104        1      81       72      18      40 26.6    0.283  24
108        4     144       58      28     140 29.5    0.287  37
109        3      83       58      31      18 34.3    0.336  25
112        8     155       62      26     495 34.0    0.543  46
113        1      89       76      34      37 31.2    0.192  23
115        7     160       54      32     175 30.5    0.588  39
120        4      99       76      15      51 23.2    0.223  21
121        0     162       76      56     100 53.2    0.759  25
123        2     107       74      30     100 33.6    0.404  23
126        1      88       30      42      99 55.0    0.496  26
127        3     120       70      30     135 42.9    0.452  30
128        1     118       58      36      94 33.3    0.261  23
129        1     117       88      24     145 34.5    0.403  40
131        4     173       70      14     168 29.7    0.361  33
133        3     170       64      37     225 34.5    0.356  30
135        2      96       68      13      49 21.1    0.647  26
136        2     125       60      20     140 33.8    0.088  31
140        5     105       72      29     325 36.9    0.159  28
143        2     108       52      26      63 32.5    0.318  22
145        4     154       62      31     284 32.8    0.237  23
148        2     106       64      35     119 30.5    1.400  34
151        1     136       74      50     204 37.4    0.399  24
153        9     156       86      28     155 34.3    1.189  42
154        1     153       82      42     485 40.6    0.687  23
158        1     109       56      21     135 25.2    0.833  23
159        2      88       74      19      53 29.0    0.229  22
160       17     163       72      41     114 40.9    0.817  47
162        7     102       74      40     105 37.2    0.204  45
163        0     114       80      34     285 44.2    0.167  27
172        6     134       70      23     130 35.4    0.542  29
174        1      79       60      42      48 43.5    0.678  23
175        2      75       64      24      55 29.7    0.370  33
178        0     129      110      46     130 67.1    0.319  26
182        0     119       64      18      92 34.9    0.725  23
187        8     181       68      36     495 30.1    0.615  60
188        1     128       98      41      58 32.0    1.321  33
189        8     109       76      39     114 27.9    0.640  31
190        5     139       80      35     160 31.6    0.361  25
192        9     123       70      44      94 33.1    0.374  40
196        5     158       84      41     210 39.4    0.395  29
198        3     107       62      13      48 22.9    0.678  23
199        4     109       64      44      99 34.8    0.905  26
200        4     148       60      27     318 30.9    0.150  29
204        2      99       70      16      44 20.4    0.235  27
205        6     103       72      32     190 37.7    0.324  55
214        0     140       65      26     130 42.6    0.431  24
215        9     112       82      32     175 34.2    0.260  36
216       12     151       70      40     271 41.8    0.742  38
218        6     125       68      30     120 30.0    0.464  32
221        0     177       60      29     478 34.6    1.072  21
225        1     100       66      15      56 23.6    0.666  26
226        1      87       78      27      32 34.6    0.101  22
229        4     197       70      39     744 36.7    2.329  31
230        0     117       80      31      53 45.2    0.089  24
232        6     134       80      37     370 46.2    0.238  46
233        1      79       80      25      37 25.4    0.583  22
235        3      74       68      28      45 29.7    0.293  23
237        7     181       84      21     192 35.9    0.586  51
242        4      91       70      32      88 33.1    0.446  22
244        6     119       50      22     176 27.1    1.318  33
245        2     146       76      35     194 38.2    0.329  29
248        0     165       90      33     680 52.3    0.427  23
249        9     124       70      33     402 35.4    0.282  34
255       12      92       62       7     258 27.6    0.926  44
259        1     193       50      16     375 25.9    0.655  24
260       11     155       76      28     150 33.3    1.353  51
261        3     191       68      15     130 30.9    0.299  34
266        5      96       74      18      67 33.6    0.997  43
272        2     108       62      32      56 25.2    0.128  21
274        1      71       78      50      45 33.2    0.422  21
276        2     100       70      52      57 40.5    0.677  25
280        2     108       62      10     278 25.3    0.881  22
282       10     129       76      28     122 35.9    0.280  39
283        7     133       88      15     155 32.4    0.262  37
286        7     136       74      26     135 26.0    0.647  51
287        5     155       84      44     545 38.7    0.619  34
290        5     108       72      43      75 36.1    0.263  33
291        0      78       88      29      40 36.9    0.434  21
292        0     107       62      30      74 36.6    0.757  25
293        2     128       78      37     182 43.3    1.224  31
294        1     128       48      45     194 40.5    0.613  24
296        6     151       62      31     120 35.5    0.692  28
297        2     146       70      38     360 28.0    0.337  29
298        0     126       84      29     215 30.7    0.520  24
306        2     120       76      37     105 39.7    0.215  29
307       10     161       68      23     132 25.5    0.326  47
308        0     137       68      14     148 24.8    0.143  21
310        2     124       68      28     205 32.9    0.875  30
312        0     106       70      37     148 39.4    0.605  22
313        2     155       74      17      96 26.6    0.433  27
 [ reached 'max' / getOption("max.print") -- omitted 189 rows ]

$usekernel
[1] FALSE

$varnames
[1] "pregnant" "glucose"  "pressure" "triceps"  "insulin"  "mass"     "pedigree" "age"     

attr(,"class")
[1] "NaiveBayes"
# Make predictions
prediction.naive <- model.naive %>% predict(test.pima.indians)
Numerical 0 probability for all classes with observation 1Numerical 0 probability for all classes with observation 2Numerical 0 probability for all classes with observation 3Numerical 0 probability for all classes with observation 4Numerical 0 probability for all classes with observation 5Numerical 0 probability for all classes with observation 6Numerical 0 probability for all classes with observation 7Numerical 0 probability for all classes with observation 8Numerical 0 probability for all classes with observation 9Numerical 0 probability for all classes with observation 10Numerical 0 probability for all classes with observation 11Numerical 0 probability for all classes with observation 12Numerical 0 probability for all classes with observation 13Numerical 0 probability for all classes with observation 14Numerical 0 probability for all classes with observation 15Numerical 0 probability for all classes with observation 16Numerical 0 probability for all classes with observation 17Numerical 0 probability for all classes with observation 18Numerical 0 probability for all classes with observation 19Numerical 0 probability for all classes with observation 20Numerical 0 probability for all classes with observation 21Numerical 0 probability for all classes with observation 22Numerical 0 probability for all classes with observation 23Numerical 0 probability for all classes with observation 24Numerical 0 probability for all classes with observation 25Numerical 0 probability for all classes with observation 26Numerical 0 probability for all classes with observation 27Numerical 0 probability for all classes with observation 28Numerical 0 probability for all classes with observation 29Numerical 0 probability for all classes with observation 30Numerical 0 probability for all classes with observation 31Numerical 0 probability for all classes with observation 32Numerical 0 probability for all classes with observation 33Numerical 0 probability for all classes with observation 34Numerical 0 probability for all classes with observation 35Numerical 0 probability for all classes with observation 36Numerical 0 probability for all classes with observation 37Numerical 0 probability for all classes with observation 38Numerical 0 probability for all classes with observation 39Numerical 0 probability for all classes with observation 40Numerical 0 probability for all classes with observation 41Numerical 0 probability for all classes with observation 42Numerical 0 probability for all classes with observation 43Numerical 0 probability for all classes with observation 44Numerical 0 probability for all classes with observation 45Numerical 0 probability for all classes with observation 46Numerical 0 probability for all classes with observation 47Numerical 0 probability for all classes with observation 48Numerical 0 probability for all classes with observation 49Numerical 0 probability for all classes with observation 50Numerical 0 probability for all classes with observation 51Numerical 0 probability for all classes with observation 52Numerical 0 probability for all classes with observation 53Numerical 0 probability for all classes with observation 54Numerical 0 probability for all classes with observation 55Numerical 0 probability for all classes with observation 56Numerical 0 probability for all classes with observation 57Numerical 0 probability for all classes with observation 58Numerical 0 probability for all classes with observation 59Numerical 0 probability for all classes with observation 60Numerical 0 probability for all classes with observation 61Numerical 0 probability for all classes with observation 62Numerical 0 probability for all classes with observation 63Numerical 0 probability for all classes with observation 64Numerical 0 probability for all classes with observation 65Numerical 0 probability for all classes with observation 66Numerical 0 probability for all classes with observation 67Numerical 0 probability for all classes with observation 68Numerical 0 probability for all classes with observation 69Numerical 0 probability for all classes with observation 70Numerical 0 probability for all classes with observation 71Numerical 0 probability for all classes with observation 72Numerical 0 probability for all classes with observation 73Numerical 0 probability for all classes with observation 74Numerical 0 probability for all classes with observation 75Numerical 0 probability for all classes with observation 76Numerical 0 probability for all classes with observation 77Numerical 0 probability for all classes with observation 78
# Model accuracy
mean(prediction.naive$class == test.pima.indians$diabetes)
[1] 0.8205128

Using caret R package

The caet R package can automatically train the model and assess the modle accuracy using k-fold cross-validation

library(klaR)

set.seed(123)

model.naive2 <- train(diabetes~., data = train.pima.indians, method = "nb", trControl = trainControl("cv", number = 10))
Numerical 0 probability for all classes with observation 1Numerical 0 probability for all classes with observation 2Numerical 0 probability for all classes with observation 3Numerical 0 probability for all classes with observation 4Numerical 0 probability for all classes with observation 5Numerical 0 probability for all classes with observation 6Numerical 0 probability for all classes with observation 7Numerical 0 probability for all classes with observation 8Numerical 0 probability for all classes with observation 9Numerical 0 probability for all classes with observation 10Numerical 0 probability for all classes with observation 11Numerical 0 probability for all classes with observation 12Numerical 0 probability for all classes with observation 13Numerical 0 probability for all classes with observation 14Numerical 0 probability for all classes with observation 15Numerical 0 probability for all classes with observation 16Numerical 0 probability for all classes with observation 17Numerical 0 probability for all classes with observation 18Numerical 0 probability for all classes with observation 19Numerical 0 probability for all classes with observation 20Numerical 0 probability for all classes with observation 21Numerical 0 probability for all classes with observation 22Numerical 0 probability for all classes with observation 23Numerical 0 probability for all classes with observation 24Numerical 0 probability for all classes with observation 25Numerical 0 probability for all classes with observation 26Numerical 0 probability for all classes with observation 27Numerical 0 probability for all classes with observation 28Numerical 0 probability for all classes with observation 29Numerical 0 probability for all classes with observation 30Numerical 0 probability for all classes with observation 31Numerical 0 probability for all classes with observation 1Numerical 0 probability for all classes with observation 2Numerical 0 probability for all classes with observation 3Numerical 0 probability for all classes with observation 4Numerical 0 probability for all classes with observation 5Numerical 0 probability for all classes with observation 6Numerical 0 probability for all classes with observation 7Numerical 0 probability for all classes with observation 8Numerical 0 probability for all classes with observation 9Numerical 0 probability for all classes with observation 10Numerical 0 probability for all classes with observation 11Numerical 0 probability for all classes with observation 12Numerical 0 probability for all classes with observation 13Numerical 0 probability for all classes with observation 14Numerical 0 probability for all classes with observation 15Numerical 0 probability for all classes with observation 16Numerical 0 probability for all classes with observation 17Numerical 0 probability for all classes with observation 18Numerical 0 probability for all classes with observation 19Numerical 0 probability for all classes with observation 20Numerical 0 probability for all classes with observation 21Numerical 0 probability for all classes with observation 22Numerical 0 probability for all classes with observation 23Numerical 0 probability for all classes with observation 24Numerical 0 probability for all classes with observation 25Numerical 0 probability for all classes with observation 26Numerical 0 probability for all classes with observation 27Numerical 0 probability for all classes with observation 28Numerical 0 probability for all classes with observation 29Numerical 0 probability for all classes with observation 30Numerical 0 probability for all classes with observation 31Numerical 0 probability for all classes with observation 1Numerical 0 probability for all classes with observation 2Numerical 0 probability for all classes with observation 3Numerical 0 probability for all classes with observation 4Numerical 0 probability for all classes with observation 5Numerical 0 probability for all classes with observation 6Numerical 0 probability for all classes with observation 7Numerical 0 probability for all classes with observation 8Numerical 0 probability for all classes with observation 9Numerical 0 probability for all classes with observation 10Numerical 0 probability for all classes with observation 11Numerical 0 probability for all classes with observation 12Numerical 0 probability for all classes with observation 13Numerical 0 probability for all classes with observation 14Numerical 0 probability for all classes with observation 15Numerical 0 probability for all classes with observation 16Numerical 0 probability for all classes with observation 17Numerical 0 probability for all classes with observation 18Numerical 0 probability for all classes with observation 19Numerical 0 probability for all classes with observation 20Numerical 0 probability for all classes with observation 21Numerical 0 probability for all classes with observation 22Numerical 0 probability for all classes with observation 23Numerical 0 probability for all classes with observation 24Numerical 0 probability for all classes with observation 25Numerical 0 probability for all classes with observation 26Numerical 0 probability for all classes with observation 27Numerical 0 probability for all classes with observation 28Numerical 0 probability for all classes with observation 29Numerical 0 probability for all classes with observation 30Numerical 0 probability for all classes with observation 31Numerical 0 probability for all classes with observation 1Numerical 0 probability for all classes with observation 2Numerical 0 probability for all classes with observation 3Numerical 0 probability for all classes with observation 4Numerical 0 probability for all classes with observation 5Numerical 0 probability for all classes with observation 6Numerical 0 probability for all classes with observation 7Numerical 0 probability for all classes with observation 8Numerical 0 probability for all classes with observation 9Numerical 0 probability for all classes with observation 10Numerical 0 probability for all classes with observation 11Numerical 0 probability for all classes with observation 12Numerical 0 probability for all classes with observation 13Numerical 0 probability for all classes with observation 14Numerical 0 probability for all classes with observation 15Numerical 0 probability for all classes with observation 16Numerical 0 probability for all classes with observation 17Numerical 0 probability for all classes with observation 18Numerical 0 probability for all classes with observation 19Numerical 0 probability for all classes with observation 20Numerical 0 probability for all classes with observation 21Numerical 0 probability for all classes with observation 22Numerical 0 probability for all classes with observation 23Numerical 0 probability for all classes with observation 24Numerical 0 probability for all classes with observation 25Numerical 0 probability for all classes with observation 26Numerical 0 probability for all classes with observation 27Numerical 0 probability for all classes with observation 28Numerical 0 probability for all classes with observation 29Numerical 0 probability for all classes with observation 30Numerical 0 probability for all classes with observation 31Numerical 0 probability for all classes with observation 1Numerical 0 probability for all classes with observation 2Numerical 0 probability for all classes with observation 3Numerical 0 probability for all classes with observation 4Numerical 0 probability for all classes with observation 5Numerical 0 probability for all classes with observation 6Numerical 0 probability for all classes with observation 7Numerical 0 probability for all classes with observation 8Numerical 0 probability for all classes with observation 9Numerical 0 probability for all classes with observation 10Numerical 0 probability for all classes with observation 11Numerical 0 probability for all classes with observation 12Numerical 0 probability for all classes with observation 13Numerical 0 probability for all classes with observation 14Numerical 0 probability for all classes with observation 15Numerical 0 probability for all classes with observation 16Numerical 0 probability for all classes with observation 17Numerical 0 probability for all classes with observation 18Numerical 0 probability for all classes with observation 19Numerical 0 probability for all classes with observation 20Numerical 0 probability for all classes with observation 21Numerical 0 probability for all classes with observation 22Numerical 0 probability for all classes with observation 23Numerical 0 probability for all classes with observation 24Numerical 0 probability for all classes with observation 25Numerical 0 probability for all classes with observation 26Numerical 0 probability for all classes with observation 27Numerical 0 probability for all classes with observation 28Numerical 0 probability for all classes with observation 29Numerical 0 probability for all classes with observation 30Numerical 0 probability for all classes with observation 31Numerical 0 probability for all classes with observation 32Numerical 0 probability for all classes with observation 1Numerical 0 probability for all classes with observation 2Numerical 0 probability for all classes with observation 3Numerical 0 probability for all classes with observation 4Numerical 0 probability for all classes with observation 5Numerical 0 probability for all classes with observation 6Numerical 0 probability for all classes with observation 7Numerical 0 probability for all classes with observation 8Numerical 0 probability for all classes with observation 9Numerical 0 probability for all classes with observation 10Numerical 0 probability for all classes with observation 11Numerical 0 probability for all classes with observation 12Numerical 0 probability for all classes with observation 13Numerical 0 probability for all classes with observation 14Numerical 0 probability for all classes with observation 15Numerical 0 probability for all classes with observation 16Numerical 0 probability for all classes with observation 17Numerical 0 probability for all classes with observation 18Numerical 0 probability for all classes with observation 19Numerical 0 probability for all classes with observation 20Numerical 0 probability for all classes with observation 21Numerical 0 probability for all classes with observation 22Numerical 0 probability for all classes with observation 23Numerical 0 probability for all classes with observation 24Numerical 0 probability for all classes with observation 25Numerical 0 probability for all classes with observation 26Numerical 0 probability for all classes with observation 27Numerical 0 probability for all classes with observation 28Numerical 0 probability for all classes with observation 29Numerical 0 probability for all classes with observation 30Numerical 0 probability for all classes with observation 31Numerical 0 probability for all classes with observation 32Numerical 0 probability for all classes with observation 1Numerical 0 probability for all classes with observation 2Numerical 0 probability for all classes with observation 3Numerical 0 probability for all classes with observation 4Numerical 0 probability for all classes with observation 5Numerical 0 probability for all classes with observation 6Numerical 0 probability for all classes with observation 7Numerical 0 probability for all classes with observation 8Numerical 0 probability for all classes with observation 9Numerical 0 probability for all classes with observation 10Numerical 0 probability for all classes with observation 11Numerical 0 probability for all classes with observation 12Numerical 0 probability for all classes with observation 13Numerical 0 probability for all classes with observation 14Numerical 0 probability for all classes with observation 15Numerical 0 probability for all classes with observation 16Numerical 0 probability for all classes with observation 17Numerical 0 probability for all classes with observation 18Numerical 0 probability for all classes with observation 19Numerical 0 probability for all classes with observation 20Numerical 0 probability for all classes with observation 21Numerical 0 probability for all classes with observation 22Numerical 0 probability for all classes with observation 23Numerical 0 probability for all classes with observation 24Numerical 0 probability for all classes with observation 25Numerical 0 probability for all classes with observation 26Numerical 0 probability for all classes with observation 27Numerical 0 probability for all classes with observation 28Numerical 0 probability for all classes with observation 29Numerical 0 probability for all classes with observation 30Numerical 0 probability for all classes with observation 31Numerical 0 probability for all classes with observation 1Numerical 0 probability for all classes with observation 2Numerical 0 probability for all classes with observation 3Numerical 0 probability for all classes with observation 4Numerical 0 probability for all classes with observation 5Numerical 0 probability for all classes with observation 6Numerical 0 probability for all classes with observation 7Numerical 0 probability for all classes with observation 8Numerical 0 probability for all classes with observation 9Numerical 0 probability for all classes with observation 10Numerical 0 probability for all classes with observation 11Numerical 0 probability for all classes with observation 12Numerical 0 probability for all classes with observation 13Numerical 0 probability for all classes with observation 14Numerical 0 probability for all classes with observation 15Numerical 0 probability for all classes with observation 16Numerical 0 probability for all classes with observation 17Numerical 0 probability for all classes with observation 18Numerical 0 probability for all classes with observation 19Numerical 0 probability for all classes with observation 20Numerical 0 probability for all classes with observation 21Numerical 0 probability for all classes with observation 22Numerical 0 probability for all classes with observation 23Numerical 0 probability for all classes with observation 24Numerical 0 probability for all classes with observation 25Numerical 0 probability for all classes with observation 26Numerical 0 probability for all classes with observation 27Numerical 0 probability for all classes with observation 28Numerical 0 probability for all classes with observation 29Numerical 0 probability for all classes with observation 30Numerical 0 probability for all classes with observation 31Numerical 0 probability for all classes with observation 1Numerical 0 probability for all classes with observation 2Numerical 0 probability for all classes with observation 3Numerical 0 probability for all classes with observation 4Numerical 0 probability for all classes with observation 5Numerical 0 probability for all classes with observation 6Numerical 0 probability for all classes with observation 7Numerical 0 probability for all classes with observation 8Numerical 0 probability for all classes with observation 9Numerical 0 probability for all classes with observation 10Numerical 0 probability for all classes with observation 11Numerical 0 probability for all classes with observation 12Numerical 0 probability for all classes with observation 13Numerical 0 probability for all classes with observation 14Numerical 0 probability for all classes with observation 15Numerical 0 probability for all classes with observation 16Numerical 0 probability for all classes with observation 17Numerical 0 probability for all classes with observation 18Numerical 0 probability for all classes with observation 19Numerical 0 probability for all classes with observation 20Numerical 0 probability for all classes with observation 21Numerical 0 probability for all classes with observation 22Numerical 0 probability for all classes with observation 23Numerical 0 probability for all classes with observation 24Numerical 0 probability for all classes with observation 25Numerical 0 probability for all classes with observation 26Numerical 0 probability for all classes with observation 27Numerical 0 probability for all classes with observation 28Numerical 0 probability for all classes with observation 29Numerical 0 probability for all classes with observation 30Numerical 0 probability for all classes with observation 31Numerical 0 probability for all classes with observation 1Numerical 0 probability for all classes with observation 2Numerical 0 probability for all classes with observation 3Numerical 0 probability for all classes with observation 4Numerical 0 probability for all classes with observation 5Numerical 0 probability for all classes with observation 6Numerical 0 probability for all classes with observation 7Numerical 0 probability for all classes with observation 8Numerical 0 probability for all classes with observation 9Numerical 0 probability for all classes with observation 10Numerical 0 probability for all classes with observation 11Numerical 0 probability for all classes with observation 12Numerical 0 probability for all classes with observation 13Numerical 0 probability for all classes with observation 14Numerical 0 probability for all classes with observation 15Numerical 0 probability for all classes with observation 16Numerical 0 probability for all classes with observation 17Numerical 0 probability for all classes with observation 18Numerical 0 probability for all classes with observation 19Numerical 0 probability for all classes with observation 20Numerical 0 probability for all classes with observation 21Numerical 0 probability for all classes with observation 22Numerical 0 probability for all classes with observation 23Numerical 0 probability for all classes with observation 24Numerical 0 probability for all classes with observation 25Numerical 0 probability for all classes with observation 26Numerical 0 probability for all classes with observation 27Numerical 0 probability for all classes with observation 28Numerical 0 probability for all classes with observation 29Numerical 0 probability for all classes with observation 30Numerical 0 probability for all classes with observation 31Numerical 0 probability for all classes with observation 1Numerical 0 probability for all classes with observation 2Numerical 0 probability for all classes with observation 3Numerical 0 probability for all classes with observation 4Numerical 0 probability for all classes with observation 5Numerical 0 probability for all classes with observation 6Numerical 0 probability for all classes with observation 7Numerical 0 probability for all classes with observation 8Numerical 0 probability for all classes with observation 9Numerical 0 probability for all classes with observation 10Numerical 0 probability for all classes with observation 11Numerical 0 probability for all classes with observation 12Numerical 0 probability for all classes with observation 13Numerical 0 probability for all classes with observation 14Numerical 0 probability for all classes with observation 15Numerical 0 probability for all classes with observation 16Numerical 0 probability for all classes with observation 17Numerical 0 probability for all classes with observation 18Numerical 0 probability for all classes with observation 19Numerical 0 probability for all classes with observation 20Numerical 0 probability for all classes with observation 21Numerical 0 probability for all classes with observation 22Numerical 0 probability for all classes with observation 23Numerical 0 probability for all classes with observation 24Numerical 0 probability for all classes with observation 25Numerical 0 probability for all classes with observation 26Numerical 0 probability for all classes with observation 27Numerical 0 probability for all classes with observation 28Numerical 0 probability for all classes with observation 29Numerical 0 probability for all classes with observation 30Numerical 0 probability for all classes with observation 31Numerical 0 probability for all classes with observation 32Numerical 0 probability for all classes with observation 1Numerical 0 probability for all classes with observation 2Numerical 0 probability for all classes with observation 3Numerical 0 probability for all classes with observation 4Numerical 0 probability for all classes with observation 5Numerical 0 probability for all classes with observation 6Numerical 0 probability for all classes with observation 7Numerical 0 probability for all classes with observation 8Numerical 0 probability for all classes with observation 9Numerical 0 probability for all classes with observation 10Numerical 0 probability for all classes with observation 11Numerical 0 probability for all classes with observation 12Numerical 0 probability for all classes with observation 13Numerical 0 probability for all classes with observation 14Numerical 0 probability for all classes with observation 15Numerical 0 probability for all classes with observation 16Numerical 0 probability for all classes with observation 17Numerical 0 probability for all classes with observation 18Numerical 0 probability for all classes with observation 19Numerical 0 probability for all classes with observation 20Numerical 0 probability for all classes with observation 21Numerical 0 probability for all classes with observation 22Numerical 0 probability for all classes with observation 23Numerical 0 probability for all classes with observation 24Numerical 0 probability for all classes with observation 25Numerical 0 probability for all classes with observation 26Numerical 0 probability for all classes with observation 27Numerical 0 probability for all classes with observation 28Numerical 0 probability for all classes with observation 29Numerical 0 probability for all classes with observation 30Numerical 0 probability for all classes with observation 31Numerical 0 probability for all classes with observation 32Numerical 0 probability for all classes with observation 1Numerical 0 probability for all classes with observation 2Numerical 0 probability for all classes with observation 3Numerical 0 probability for all classes with observation 4Numerical 0 probability for all classes with observation 5Numerical 0 probability for all classes with observation 6Numerical 0 probability for all classes with observation 7Numerical 0 probability for all classes with observation 8Numerical 0 probability for all classes with observation 9Numerical 0 probability for all classes with observation 10Numerical 0 probability for all classes with observation 11Numerical 0 probability for all classes with observation 12Numerical 0 probability for all classes with observation 13Numerical 0 probability for all classes with observation 14Numerical 0 probability for all classes with observation 15Numerical 0 probability for all classes with observation 16Numerical 0 probability for all classes with observation 17Numerical 0 probability for all classes with observation 18Numerical 0 probability for all classes with observation 19Numerical 0 probability for all classes with observation 20Numerical 0 probability for all classes with observation 21Numerical 0 probability for all classes with observation 22Numerical 0 probability for all classes with observation 23Numerical 0 probability for all classes with observation 24Numerical 0 probability for all classes with observation 25Numerical 0 probability for all classes with observation 26Numerical 0 probability for all classes with observation 27Numerical 0 probability for all classes with observation 28Numerical 0 probability for all classes with observation 29Numerical 0 probability for all classes with observation 30Numerical 0 probability for all classes with observation 31Numerical 0 probability for all classes with observation 32Numerical 0 probability for all classes with observation 1Numerical 0 probability for all classes with observation 2Numerical 0 probability for all classes with observation 3Numerical 0 probability for all classes with observation 4Numerical 0 probability for all classes with observation 5Numerical 0 probability for all classes with observation 6Numerical 0 probability for all classes with observation 7Numerical 0 probability for all classes with observation 8Numerical 0 probability for all classes with observation 9Numerical 0 probability for all classes with observation 10Numerical 0 probability for all classes with observation 11Numerical 0 probability for all classes with observation 12Numerical 0 probability for all classes with observation 13Numerical 0 probability for all classes with observation 14Numerical 0 probability for all classes with observation 15Numerical 0 probability for all classes with observation 16Numerical 0 probability for all classes with observation 17Numerical 0 probability for all classes with observation 18Numerical 0 probability for all classes with observation 19Numerical 0 probability for all classes with observation 20Numerical 0 probability for all classes with observation 21Numerical 0 probability for all classes with observation 22Numerical 0 probability for all classes with observation 23Numerical 0 probability for all classes with observation 24Numerical 0 probability for all classes with observation 25Numerical 0 probability for all classes with observation 26Numerical 0 probability for all classes with observation 27Numerical 0 probability for all classes with observation 28Numerical 0 probability for all classes with observation 29Numerical 0 probability for all classes with observation 30Numerical 0 probability for all classes with observation 31Numerical 0 probability for all classes with observation 32Numerical 0 probability for all classes with observation 1Numerical 0 probability for all classes with observation 2Numerical 0 probability for all classes with observation 3Numerical 0 probability for all classes with observation 4Numerical 0 probability for all classes with observation 5Numerical 0 probability for all classes with observation 6Numerical 0 probability for all classes with observation 7Numerical 0 probability for all classes with observation 8Numerical 0 probability for all classes with observation 9Numerical 0 probability for all classes with observation 10Numerical 0 probability for all classes with observation 11Numerical 0 probability for all classes with observation 12Numerical 0 probability for all classes with observation 13Numerical 0 probability for all classes with observation 14Numerical 0 probability for all classes with observation 15Numerical 0 probability for all classes with observation 16Numerical 0 probability for all classes with observation 17Numerical 0 probability for all classes with observation 18Numerical 0 probability for all classes with observation 19Numerical 0 probability for all classes with observation 20Numerical 0 probability for all classes with observation 21Numerical 0 probability for all classes with observation 22Numerical 0 probability for all classes with observation 23Numerical 0 probability for all classes with observation 24Numerical 0 probability for all classes with observation 25Numerical 0 probability for all classes with observation 26Numerical 0 probability for all classes with observation 27Numerical 0 probability for all classes with observation 28Numerical 0 probability for all classes with observation 29Numerical 0 probability for all classes with observation 30Numerical 0 probability for all classes with observation 31Numerical 0 probability for all classes with observation 1Numerical 0 probability for all classes with observation 2Numerical 0 probability for all classes with observation 3Numerical 0 probability for all classes with observation 4Numerical 0 probability for all classes with observation 5Numerical 0 probability for all classes with observation 6Numerical 0 probability for all classes with observation 7Numerical 0 probability for all classes with observation 8Numerical 0 probability for all classes with observation 9Numerical 0 probability for all classes with observation 10Numerical 0 probability for all classes with observation 11Numerical 0 probability for all classes with observation 12Numerical 0 probability for all classes with observation 13Numerical 0 probability for all classes with observation 14Numerical 0 probability for all classes with observation 15Numerical 0 probability for all classes with observation 16Numerical 0 probability for all classes with observation 17Numerical 0 probability for all classes with observation 18Numerical 0 probability for all classes with observation 19Numerical 0 probability for all classes with observation 20Numerical 0 probability for all classes with observation 21Numerical 0 probability for all classes with observation 22Numerical 0 probability for all classes with observation 23Numerical 0 probability for all classes with observation 24Numerical 0 probability for all classes with observation 25Numerical 0 probability for all classes with observation 26Numerical 0 probability for all classes with observation 27Numerical 0 probability for all classes with observation 28Numerical 0 probability for all classes with observation 29Numerical 0 probability for all classes with observation 30Numerical 0 probability for all classes with observation 31Numerical 0 probability for all classes with observation 1Numerical 0 probability for all classes with observation 2Numerical 0 probability for all classes with observation 3Numerical 0 probability for all classes with observation 4Numerical 0 probability for all classes with observation 5Numerical 0 probability for all classes with observation 6Numerical 0 probability for all classes with observation 7Numerical 0 probability for all classes with observation 8Numerical 0 probability for all classes with observation 9Numerical 0 probability for all classes with observation 10Numerical 0 probability for all classes with observation 11Numerical 0 probability for all classes with observation 12Numerical 0 probability for all classes with observation 13Numerical 0 probability for all classes with observation 14Numerical 0 probability for all classes with observation 15Numerical 0 probability for all classes with observation 16Numerical 0 probability for all classes with observation 17Numerical 0 probability for all classes with observation 18Numerical 0 probability for all classes with observation 19Numerical 0 probability for all classes with observation 20Numerical 0 probability for all classes with observation 21Numerical 0 probability for all classes with observation 22Numerical 0 probability for all classes with observation 23Numerical 0 probability for all classes with observation 24Numerical 0 probability for all classes with observation 25Numerical 0 probability for all classes with observation 26Numerical 0 probability for all classes with observation 27Numerical 0 probability for all classes with observation 28Numerical 0 probability for all classes with observation 29Numerical 0 probability for all classes with observation 30Numerical 0 probability for all classes with observation 31Numerical 0 probability for all classes with observation 1Numerical 0 probability for all classes with observation 2Numerical 0 probability for all classes with observation 3Numerical 0 probability for all classes with observation 4Numerical 0 probability for all classes with observation 5Numerical 0 probability for all classes with observation 6Numerical 0 probability for all classes with observation 7Numerical 0 probability for all classes with observation 8Numerical 0 probability for all classes with observation 9Numerical 0 probability for all classes with observation 10Numerical 0 probability for all classes with observation 11Numerical 0 probability for all classes with observation 12Numerical 0 probability for all classes with observation 13Numerical 0 probability for all classes with observation 14Numerical 0 probability for all classes with observation 15Numerical 0 probability for all classes with observation 16Numerical 0 probability for all classes with observation 17Numerical 0 probability for all classes with observation 18Numerical 0 probability for all classes with observation 19Numerical 0 probability for all classes with observation 20Numerical 0 probability for all classes with observation 21Numerical 0 probability for all classes with observation 22Numerical 0 probability for all classes with observation 23Numerical 0 probability for all classes with observation 24Numerical 0 probability for all classes with observation 25Numerical 0 probability for all classes with observation 26Numerical 0 probability for all classes with observation 27Numerical 0 probability for all classes with observation 28Numerical 0 probability for all classes with observation 29Numerical 0 probability for all classes with observation 30Numerical 0 probability for all classes with observation 31Numerical 0 probability for all classes with observation 1Numerical 0 probability for all classes with observation 2Numerical 0 probability for all classes with observation 3Numerical 0 probability for all classes with observation 4Numerical 0 probability for all classes with observation 5Numerical 0 probability for all classes with observation 6Numerical 0 probability for all classes with observation 7Numerical 0 probability for all classes with observation 8Numerical 0 probability for all classes with observation 9Numerical 0 probability for all classes with observation 10Numerical 0 probability for all classes with observation 11Numerical 0 probability for all classes with observation 12Numerical 0 probability for all classes with observation 13Numerical 0 probability for all classes with observation 14Numerical 0 probability for all classes with observation 15Numerical 0 probability for all classes with observation 16Numerical 0 probability for all classes with observation 17Numerical 0 probability for all classes with observation 18Numerical 0 probability for all classes with observation 19Numerical 0 probability for all classes with observation 20Numerical 0 probability for all classes with observation 21Numerical 0 probability for all classes with observation 22Numerical 0 probability for all classes with observation 23Numerical 0 probability for all classes with observation 24Numerical 0 probability for all classes with observation 25Numerical 0 probability for all classes with observation 26Numerical 0 probability for all classes with observation 27Numerical 0 probability for all classes with observation 28Numerical 0 probability for all classes with observation 29Numerical 0 probability for all classes with observation 30Numerical 0 probability for all classes with observation 31Numerical 0 probability for all classes with observation 32Numerical 0 probability for all classes with observation 1Numerical 0 probability for all classes with observation 2Numerical 0 probability for all classes with observation 3Numerical 0 probability for all classes with observation 4Numerical 0 probability for all classes with observation 5Numerical 0 probability for all classes with observation 6Numerical 0 probability for all classes with observation 7Numerical 0 probability for all classes with observation 8Numerical 0 probability for all classes with observation 9Numerical 0 probability for all classes with observation 10Numerical 0 probability for all classes with observation 11Numerical 0 probability for all classes with observation 12Numerical 0 probability for all classes with observation 13Numerical 0 probability for all classes with observation 14Numerical 0 probability for all classes with observation 15Numerical 0 probability for all classes with observation 16Numerical 0 probability for all classes with observation 17Numerical 0 probability for all classes with observation 18Numerical 0 probability for all classes with observation 19Numerical 0 probability for all classes with observation 20Numerical 0 probability for all classes with observation 21Numerical 0 probability for all classes with observation 22Numerical 0 probability for all classes with observation 23Numerical 0 probability for all classes with observation 24Numerical 0 probability for all classes with observation 25Numerical 0 probability for all classes with observation 26Numerical 0 probability for all classes with observation 27Numerical 0 probability for all classes with observation 28Numerical 0 probability for all classes with observation 29Numerical 0 probability for all classes with observation 30Numerical 0 probability for all classes with observation 31Numerical 0 probability for all classes with observation 32
# make predictions
predicted.classes.naive2 <- model.naive2 %>% predict(test.pima.indians)
Numerical 0 probability for all classes with observation 1Numerical 0 probability for all classes with observation 2Numerical 0 probability for all classes with observation 3Numerical 0 probability for all classes with observation 4Numerical 0 probability for all classes with observation 5Numerical 0 probability for all classes with observation 6Numerical 0 probability for all classes with observation 7Numerical 0 probability for all classes with observation 8Numerical 0 probability for all classes with observation 9Numerical 0 probability for all classes with observation 10Numerical 0 probability for all classes with observation 11Numerical 0 probability for all classes with observation 12Numerical 0 probability for all classes with observation 13Numerical 0 probability for all classes with observation 14Numerical 0 probability for all classes with observation 15Numerical 0 probability for all classes with observation 16Numerical 0 probability for all classes with observation 17Numerical 0 probability for all classes with observation 18Numerical 0 probability for all classes with observation 19Numerical 0 probability for all classes with observation 20Numerical 0 probability for all classes with observation 21Numerical 0 probability for all classes with observation 22Numerical 0 probability for all classes with observation 23Numerical 0 probability for all classes with observation 24Numerical 0 probability for all classes with observation 25Numerical 0 probability for all classes with observation 26Numerical 0 probability for all classes with observation 27Numerical 0 probability for all classes with observation 28Numerical 0 probability for all classes with observation 29Numerical 0 probability for all classes with observation 30Numerical 0 probability for all classes with observation 31Numerical 0 probability for all classes with observation 32Numerical 0 probability for all classes with observation 33Numerical 0 probability for all classes with observation 34Numerical 0 probability for all classes with observation 35Numerical 0 probability for all classes with observation 36Numerical 0 probability for all classes with observation 37Numerical 0 probability for all classes with observation 38Numerical 0 probability for all classes with observation 39Numerical 0 probability for all classes with observation 40Numerical 0 probability for all classes with observation 41Numerical 0 probability for all classes with observation 42Numerical 0 probability for all classes with observation 43Numerical 0 probability for all classes with observation 44Numerical 0 probability for all classes with observation 45Numerical 0 probability for all classes with observation 46Numerical 0 probability for all classes with observation 47Numerical 0 probability for all classes with observation 48Numerical 0 probability for all classes with observation 49Numerical 0 probability for all classes with observation 50Numerical 0 probability for all classes with observation 51Numerical 0 probability for all classes with observation 52Numerical 0 probability for all classes with observation 53Numerical 0 probability for all classes with observation 54Numerical 0 probability for all classes with observation 55Numerical 0 probability for all classes with observation 56Numerical 0 probability for all classes with observation 57Numerical 0 probability for all classes with observation 58Numerical 0 probability for all classes with observation 59Numerical 0 probability for all classes with observation 60Numerical 0 probability for all classes with observation 61Numerical 0 probability for all classes with observation 62Numerical 0 probability for all classes with observation 63Numerical 0 probability for all classes with observation 64Numerical 0 probability for all classes with observation 65Numerical 0 probability for all classes with observation 66Numerical 0 probability for all classes with observation 67Numerical 0 probability for all classes with observation 68Numerical 0 probability for all classes with observation 69Numerical 0 probability for all classes with observation 70Numerical 0 probability for all classes with observation 71Numerical 0 probability for all classes with observation 72Numerical 0 probability for all classes with observation 73Numerical 0 probability for all classes with observation 74Numerical 0 probability for all classes with observation 75Numerical 0 probability for all classes with observation 76Numerical 0 probability for all classes with observation 77Numerical 0 probability for all classes with observation 78
# Model n accuracy

mean(predicted.classes.naive2 == test.pima.indians$diabetes)
[1] 0.8076923

Support Vector Machine

Support Vector Machine (or SVM) is a machine learning technique used for classification tasks. Briefly, SVM works by identifying the optimal decision boundary that separates data points from different groups (or classes), and then predicts the class of new observations based on the separation boundary.

Depending on the situations, the different groups might be separable by a linear straight line or by a non-linear boundary line.

Support vector machine methods can handle both linear and non-linear class boundaries. It can be used for both two-classs and multi-class classification problems.

In real life data, the separation boundary is generally nonlinear. Technically, the SVM algorithm perform a non-linear classification using what is called the kernel trick. The most commonly used kernel transformations ar e_polynomial kernel_ and radial kernel

Note that, there is also an extension of the SVM for regression, called support vector regression.

Loading required R packages

library(tidyverse)
library(caret)

Dataset

Data set: PimaIndiansDiabetes2 [in mlbench package]

# Load the data
data("PimaIndiansDiabetes2", package = "mlbench")
pima.data.cleaned <- na.omit(PimaIndiansDiabetes2)

# Split the data into training and test set
set.seed(123)

t.sample <- pima.data.cleaned$diabetes %>% createDataPartition(p = 0.8, list = FALSE)

SVM linear classifier

In the following example variables are normalized to make their scale comparable. This is automatically done before building the SVM classifier by setting the option preProcess = c(“center”, “scale”).

# Fit the model on the training set
set.seed(123)

model.svm <- train(
  diabetes~., data = train.pima.indians, method = "svmLinear",
  trContro = trainControl("cv", number = 10),
  preProcess = c("center", "scale")
)

# Make predictions on the test data
predicted.classes.svm <- model.svm %>% predict(test.pima.indians)
head(predicted.classes.svm)
[1] neg pos neg pos pos neg
Levels: neg pos
# Computed model accuracy rate
mean(predicted.classes.svm == test.pima.indians$diabetes)
[1] 0.7820513

Note that, there is a tuning parameter C, also known as Cost, that determines the possible misclassifications. It essentially imposes a penalty to the model for making an error. The higher the value of C, the less likely it is that the SVM algorithm will misclassify a point.

By default caret builds the SVM linear classifier usin g C= 1. You can check this by typing model in R console

model.svm
Support Vector Machines with Linear Kernel 

314 samples
  8 predictor
  2 classes: 'neg', 'pos' 

Pre-processing: centered (8), scaled (8) 
Resampling: Bootstrapped (25 reps) 
Summary of sample sizes: 314, 314, 314, 314, 314, 314, ... 
Resampling results:

  Accuracy   Kappa    
  0.7862095  0.4893315

Tuning parameter 'C' was held constant at a value of 1

The following R code compute SVM for a grid values of C and choose automatically the final model for predictions:

# Fit the model on the training set
set.seed(123)

model.svm2 <- train(
  diabetes ~., data = train.pima.indians, method = "svmLinear",
  trControl = trainControl("cv", number = 10),
  tuneGrid = expand.grid(C = seq(0.1, 2, length = 20)),
  preProcess = c("center", "scale")
)

# Plot model accuracy vs different values of Cost
plot(model.svm2)

# Print the best tuning parameter C that maximizes model accuracy
model.svm2$bestTune

Let’s use the best model

# Make predictions on the test data 
predicted.classess.svm2 <- model.svm2 %>% predict(test.pima.indians)

# Compute model accuracy rate
mean(predicted.classess.svm2 == test.pima.indians$diabetes)
[1] 0.7820513

SVM classifier using Non-Linear Kernel

To build a non-linear SVM classifier, we can use either polynomial kernel or radial kernel function.

  • Computing SVM using radial basis kernel
# Fit the model on the training set
set.seed(123)
model.svm.radial <- train(
  diabetes~., data = train.pima.indians, method = "svmRadial",
  trControl = trainControl("cv", number = 10),
  preProcess = c("center", "scale"),
  tuneLength = 10
)

# Print the best tuning parameter sigma and C that maximizes model accuracy

model.svm.radial$bestTune
# Make predictions on the test data
predicted.classess.svm.radial <- model.svm.radial %>% predict(test.pima.indians)
# Compute model Accuracy rate
mean(predicted.classess.svm.radial == test.pima.indians$diabetes)
[1] 0.7948718
  • Computing SVM using polynomial basis kernel:
# Fit the model on the training set
set.seed(123)
model.svm.polynomial <- train(
  diabetes~., data = train.pima.indians, method = "svmPoly",
  trControl = trainControl("cv", number = 10),
  preProcess = c("center", "scale"),
  tuneLength = 4
)

# Print the best tuning parameter sigma and C that maximizes model accuracy

model.svm.polynomial$bestTune
# Make predictions on the test data
predicted.classess.svm.polynomial <- model.svm.polynomial %>% predict(test.pima.indians)
# Compute model Accuracy rate
mean(predicted.classess.svm.polynomial == test.pima.indians$diabetes)
[1] 0.7948718

In our examples, it can be seen that the SVM classifier using non-linear kernel gives a better result compared to the linear model.

Classification Model Evaluation

After building a predictive classification model, you need to evaluate the performance of the model, that is how good the model is in predicting the outcome of new observations test data that have been not used to train the model.

The common use metrics and methods for assessing the performance of predictive classification models:

  • Average classification accuracy - representing the proportion of correctly classified observations.

  • Confusion matrix - which is 2x2 table showing four parameters, including the number of true positives, true negatives, false negatives and false positives.

  • Precision, Recall and Specificity - which are three major performance metrics describing a predictive classification model

  • ROC curve - which is a graphical summary of the overall performance of the model, showing the proportion of true positives and false positives at tall possible values of probability cutoff. The Area Under the Curve(AUC) summarizes the overall performance of the classifier.

Required R packages

  • tidyverse for easy data manipulation and visualization
  • caret for easy machine learning workflow
library(tidyverse)
library(caret)

Building a classification model

To keep things simple we’ll perform a binary classification, where the outcome variable can have only two possible values: negative vs positive.

We’ll compute an example of linear discriminant analysis model using the PimaIndiansDiabetes2 [mlbench package]

  1. Split the data into training (80%, used to build the model) and test set (20%, used to evaluate the model performance):
data("PimaIndiansDiabetes2", package = "mlbench")
pima.data.cleaned <- na.omit(PimaIndiansDiabetes2)
# Split the data into training and test set
set.seed(123)

t.sample <- pima.data.cleaned$diabetes %>% createDataPartition(p = 0.8, list = FALSE)

train.pima2 <- pima.data.cleaned[t.sample, ]
test.pima2 <- pima.data.cleaned[-t.sample, ]
  1. Fit the LDA model on the training set and make preditions on the test data:
library(MASS)
#Fit LDA

fit.lda <- lda(diabetes ~., data = train.pima2)

# Make predictions on the test data
predictions.pima2 <- predict(fit.lda, test.pima2)
preictions.probabilities2 <- predictions.pima2$posterior[,2]
predicted.classess2 <- predictions.pima2$class
observed.classes2 <- test.pima2$diabetes

Overall classification accuracy

The overall classification accuracy rate corresponds to the proportion of observations that have been correctly classified. Determining the raw classification accuracy is the first step in assessing the performance of a model.

accuracy <- mean(observed.classes2 == predicted.classess2)
accuracy
[1] 0.8076923
error <- mean(observed.classes2 != predicted.classess2)
error
[1] 0.1923077

From the output avove, the linear discrimant analysis correctly predicted the individual outcome in 81% of the cases. This is by far better than random guessing. The misclassification error rate can be calculated as 100-81 = 19%

In our example, a binary classifier can make two types of errors:

  • it can incorrectly assign an individual who is diabetes-positive to the diabetes-negative category
  • it can incorrectly assign an individual who is diabetes-negative to the diabetes-positive category.

The proportion of these two types of errors can be determined by creating a confution matrxi, which compare the predicted coutcome values agains the known outcome vaues.

Confusion matrix

The R function table() can be used to produce a confusion matrix in order to determin how many observations were correctly or incorrectly classified. It compares the observed and the predicted outcome values and shows the number of correct and incorrect predictions categorized by type of outcome.

# Confusion matrix, number of cases
table(observed.classes2, predicted.classess2)
                 predicted.classess2
observed.classes2 neg pos
              neg  48   4
              pos  11  15
# Confusion matrix, proportion of cases
table(observed.classes2, predicted.classess2) %>% prop.table() %>% round(digits = 3)
                 predicted.classess2
observed.classes2   neg   pos
              neg 0.615 0.051
              pos 0.141 0.192

The diagonal elements of the confusion matrix indicate correct predictions, while the off diagonals represent incorrect prediction. So, the correct classification rate is the sum of the number on the diagonal divided by the sample size in the test data. In our example, that is (48+15)/78 = 81%

  • True positives (d): these are cases in which we predicted the individuals would be diabetes-positive and they were .
  • True negatives (a): We predicted diabetes-negative, and the individuals were diabetes-negative
  • False positives (b): We predicted diabetes-positive, but the individuals didn’t actually hae diabetes. (Alsow known as a Type I error.)
  • False negatives (c): We predicted diabetes-negative, but they did have diabetes. (Also known as a Type II error.)

Technically the raw prediction accuracy of the model is defined as (TruePositives + TrueNegatives)/SampleSize.

Precision, Recall and Specificity

In addition to the raw classification accuracy, there are many other metrics that are widely used to examine the performance of a classification model, including:

Precision, which is the proportion of true positives among all the individuals that have been predicted to be diabetes-positive by the model. This represents the accuracy of a predicted positive outcome. Precision = TruePositives/(TruePositives + FalsePOsitives)

Sensitivity (or Recall), which is the True Positive Rate (TPR) or the proportion of identified positives among the diabetes-positive population (class = 1). Sensitivity = TruePOsitives/(TruePositives + FalseNegatives).

Specificity, which measures the True Negative Rate (TNR), that is the proportion of identified negatives among the diabetes-negative population (class = 0). Specificity = TrueNegatives/(TrueNegatives + FalseNegatives).

False POsitive Rate (FPR), which represents the proportion of identified positives among the healthy individuals (i.e. diabetes-negative). This can be seen as a false alarm. The FPR can be also calculated as 1-specificity. When positives are rare, the FPR can be high, leading to the situation where a predicted positive is most likely a negative.

confusionMatrix(observed.classes2, predicted.classess2, positive = "pos")
Confusion Matrix and Statistics

          Reference
Prediction neg pos
       neg  48   4
       pos  11  15
                                          
               Accuracy : 0.8077          
                 95% CI : (0.7027, 0.8882)
    No Information Rate : 0.7564          
    P-Value [Acc > NIR] : 0.1787          
                                          
                  Kappa : 0.5361          
                                          
 Mcnemar's Test P-Value : 0.1213          
                                          
            Sensitivity : 0.7895          
            Specificity : 0.8136          
         Pos Pred Value : 0.5769          
         Neg Pred Value : 0.9231          
             Prevalence : 0.2436          
         Detection Rate : 0.1923          
   Detection Prevalence : 0.3333          
      Balanced Accuracy : 0.8015          
                                          
       'Positive' Class : pos             
                                          

In our example, the sensitivity is ~58%, that is the proportion of diabetes-positive individuals that were correctly identified by the model as diabetes-positive. The specificity of the model is ~92%, that is the proportion of diabetes-negative individuals that were correctly identified by the model as diabetes-negative. The model precision or the proportion of positive predicted value is 79%.

ROC curve

The ROC curve (or receiver operating characteristics curve) is apopular graphical measure for assessing the performance or the accuracy of a classifier, which corresponds to the total

Computing and plotting ROC curve

The ROC analysis can be easily performed using the R package pROC

library(pROC)
res.roc <- roc(observed.classes2,preictions.probabilities2)
Setting levels: control = neg, case = pos
Setting direction: controls < cases
plot.roc(res.roc, print.auc = TRUE)

# Extract some interesting results
roc.data <- data_frame(
  thresholds = res.roc$thresholds,
  sensitivity = res.roc$sensitivities,
  specificity = res.roc$specificities
)
`data_frame()` is deprecated, use `tibble()`.
This warning is displayed once per session.
# Get the probability threshold for specificity = 0.6

roc.data %>% filter(specificity >= 0.6)
plot.roc(res.roc, print.auc = TRUE, print.thres = "best")

plot.roc(res.roc, print.thres = c(0.3,0.5,0.7))

LS0tCnRpdGxlOiAiTUwgRXNzZW50aWFscyIKb3V0cHV0OiBodG1sX25vdGVib29rCmRhdGU6IE5vdiAyOCwgMjAyMAotLS0KCiMgb3V0cHV0OgojICAgbWRfZG9jdW1lbnQ6CiMgICAgIHZhcmlhbnQ6IG1hcmtkb3duX2dpdGh1YgoKYGBge3J9CnNldHdkKCJ+L0hvbWVXb3JrL1ByYWN0aWNlIikKYGBgCgoKYGBge3J9CmRldnRvb2xzOjppbnN0YWxsX2dpdGh1Yigia2Fzc2FtYmFyYS9kYXRhcml1bSIpCmBgYAoKYGBge3J9CmRhdGEoIm1hcmtldGluZyIsIHBhY2thZ2U9ImRhdGFyaXVtIikKaGVhZChtYXJrZXRpbmcpCmBgYAoKCmBgYHtyfQpkYXRhKCJzd2lzcyIpCmhlYWQoc3dpc3MpCmBgYAoKYGBge3J9CmRhdGEoIkJvc3RvbiIsIHBhY2thZ2U9Ik1BU1MiKQpoZWFkKEJvc3RvbikKYGBgCgojIyMgQ2hhcHRlciAzIFJlZ3Jlc3Npb24KCkxvYWRpbmcgUmVxdWlyZWQgUiBwYWNrYWdlcwpgYGB7cn0KbGlicmFyeSh0aWR5dmVyc2UpCmxpYnJhcnkoY2FyZXQpCnRoZW1lX3NldCh0aGVtZV9idygpKQpgYGAKUHJlcGFyaW5nIHRoZSBkYXRhCgpgYGB7cn0KIyBMb2FkIHRoZSBkYXRhCmRhdGEoIm1hcmtldGluZyIsIHBhY2thZ2UgPSAiZGF0YXJpdW0iKQojIEluc3BlY3QgdGggZGF0YQpzYW1wbGVfbihtYXJrZXRpbmcsMykKYGBgCgpTcGxpdCB0aGUgZGF0YSBpbnRvIHRyYWluaW5nIGFuZCB0ZXN0IHNldApgYGB7cn0Kc2V0LnNlZWQoMTIzKQp0cmFpbmluZy5zYW1wbGVzIDwtIG1hcmtldGluZyRzYWxlcyAlPiUgY3JlYXRlRGF0YVBhcnRpdGlvbihwPTAuOCwgbGlzdCA9IEZBTFNFKQoKdHJhaW4uZGF0YSA8LSBtYXJrZXRpbmdbdHJhaW5pbmcuc2FtcGxlcyxdCnRlc3QuZGF0YSA8LSBtYXJrZXRpbmdbLXRyYWluaW5nLnNhbXBsZXMsXQpgYGAKYGBge3J9CnN1bW1hcnkodHJhaW4uZGF0YSkKZGltKHRyYWluLmRhdGEpCgpkaW0odGVzdC5kYXRhKQpgYGAKCkNvbXB1dGluZyBsaW5lYXIgcmVncmVzc2lvbiBpbiByCgpgYGB7cn0KbW9kZWwgPC0gbG0oc2FsZXMgfi4sIGRhdGEgPSB0cmFpbi5kYXRhKQpzdW1tYXJ5KG1vZGVsKQoKIyBNYWtlIHByZWRpY3Rpb3MKcHJlZGljdGlvbnMgPC0gbW9kZWwgJT4lIHByZWRpY3QodGVzdC5kYXRhKQoKIyBNb2RlbCBQZXJmb3JtYW5jZQojIChhKSBQcmVkaWN0aW9uIGVycm9yLCBSTVNFClJNU0UocHJlZGljdGlvbnMsIHRlc3QuZGF0YSRzYWxlcykKIyAoYikgUi1zcXVhcmUKUjIocHJlZGljdGlvbnMsIHRlc3QuZGF0YSRzYWxlcykKCmBgYAoKIyMgTm9uLUxpbmVhciBSZWdyZXNzaW9uCgpgYGB7cn0KZGF0YSgiQm9zdG9uIiwgcGFja2FnZSA9ICJNQVNTIikKc2V0LnNlZWQoMTIzKQoKaGVhZChCb3N0b24pCmBgYAoKYGBge3J9CnRyYWluaW5nMS5zYW1wbGVzIDwtIEJvc3RvbiRtZWR2ICU+JSBjcmVhdGVEYXRhUGFydGl0aW9uKHA9MC44LCBsaXN0ID0gRkFMU0UpCgp0cmFpbjEuZGF0YSA8LSBCb3N0b25bdHJhaW5pbmcuc2FtcGxlcyxdCnRlc3QxLmRhdGEgPC0gQm9zdG9uWy10cmFpbmluZy5zYW1wbGVzLCBdCgpnZ3Bsb3QodHJhaW4xLmRhdGEsIGFlcyhsc3RhdCwgbWVkdikpICsKICBnZW9tX3BvaW50KCkgKwogIHN0YXRfc21vb3RoKCkKYGBgCgpMZXQncyBzdGFydCB3aXRoIGZpdHRpbmcgYSBsaW5lYXIgcmVncmVzc2lvbiBtb2RlbApgYGB7cn0KbW9kZWwubG0gPC0gbG0obWVkdn5sc3RhdCwgZGF0YSA9IHRyYWluMS5kYXRhKQpzdW1tYXJ5KG1vZGVsLmxtKQpwcmVkaWN0aW9uIDwtIG1vZGVsLmxtICU+JSBwcmVkaWN0KHRlc3QxLmRhdGEpCgpkYXRhLmZyYW1lKFJNU0UgPSBSTVNFKHByZWRpY3Rpb24sIHRlc3QxLmRhdGEkbWVkdiksCiAgICAgICAgICAgUjIgPSBSMihwcmVkaWN0aW9uLCB0ZXN0MS5kYXRhJG1lZHYpKQpgYGAKCkxldCdzIHZpc3VhbGl6ZSB0aGUgZGF0YQpgYGB7cn0KZ2dwbG90KHRyYWluMS5kYXRhLCBhZXMobHN0YXQsIG1lZHYpKSArCiAgZ2VvbV9wb2ludCgpICsKICBzdGF0X3Ntb290aChtZXRob2QgPSBsbSwgZm9ybXVsYSA9IHkgfiB4KQpgYGAKCgojIyMgUG9seW5vbWlhbCByZWdyZXNzaW9uCgoKCm1lZHYgPSBiMCArIGIxIOKIlyBsc3RhdCArIGIyIOKIlyBsc3RhdDIKCgpgYGB7cn0KbW9kMiA8LSBsbShtZWR2IH4gbHN0YXQgKyBJKGxzdGF0KmxzdGF0KSwgZGF0YSA9IHRyYWluMS5kYXRhKQpzdW1tYXJ5KG1vZDIpCnByZWRpY3Rpb24yIDwtIG1vZDIgJT4lIHByZWRpY3QodGVzdDEuZGF0YSkKZGF0YS5mcmFtZSgKICBSTVNFID0gUk1TRShwcmVkaWN0aW9uMiwgdGVzdDEuZGF0YSRtZWR2KSwKICBSMiA9IFIyKHByZWRpY3Rpb24yLCB0ZXN0MS5kYXRhJG1lZHYpCikKbW9kMjIgPC0gbG0obWVkdiB+IHBvbHkobHN0YXQsMiksIGRhdGEgPSB0cmFpbjEuZGF0YSkKc3VtbWFyeShtb2QyMikKcHJlZGljdGlvbjIyIDwtIG1vZDIyICU+JSBwcmVkaWN0KHRlc3QxLmRhdGEpCmRhdGEuZnJhbWUoCiAgUk1TRSA9IFJNU0UocHJlZGljdGlvbjIyLCB0ZXN0MS5kYXRhJG1lZHYpLAogIFIyID0gUjIocHJlZGljdGlvbjIyLCB0ZXN0MS5kYXRhJG1lZHYpCikKYGBgCgpgYGB7cn0KZ2dwbG90KHRyYWluMS5kYXRhLCBhZXMobHN0YXQsIG1lZHYpKSArCiAgZ2VvbV9wb2ludCgpICsKICBzdGF0X3Ntb290aChtZXRob2QgPSBsbSwgZm9ybXVsYSA9IHkgfiBwb2x5KHgsMikpCmBgYAoKYGBge3J9CmdncGxvdCh0cmFpbjEuZGF0YSwgYWVzKGxzdGF0LCBtZWR2KSkgKwogIGdlb21fcG9pbnQoKSArCiAgc3RhdF9zbW9vdGgobWV0aG9kID0gbG0sIGZvcm11bGEgPSB5IH4gKHggKyBJKHgqeCkpKQpgYGAKCgpgYGB7cn0KbW9kNiA8LSBsbShtZWR2IH4gcG9seShsc3RhdCwgNiksIGRhdGEgPSB0cmFpbjEuZGF0YSkKc3VtbWFyeShtb2Q2KQpgYGAKCkFzIHdlIGNhbiBzZWUgaW4gYWJvdmUgbW9kZWwgc3VtbWFyeSB0aGUgcG9seW5vbWlhbCB0ZXJtIGJleW9uZCA1IGlzIG5vdCBzaWduaWZpY2FudAoKTGV0J3MgdmlzdWFsaXplIG1vZGVsCgpgYGB7cn0KZ2dwbG90KHRyYWluMS5kYXRhLCBhZXMobHN0YXQsIG1lZHYpKSArCiAgZ2VvbV9wb2ludCgpICsKICBzdGF0X3Ntb290aChtZXRob2QgPSBsbSwgZm9ybXVsYSA9IHkgfiBwb2x5KHgsNikpCmBgYAoKCmBgYHtyfQptb2Q1IDwtIGxtKG1lZHYgfiBwb2x5KGxzdGF0LCA1KSwgZGF0YSA9IHRyYWluMS5kYXRhKQpzdW1tYXJ5KG1vZDUpCmBgYAoKCmBgYHtyfQpnZ3Bsb3QodHJhaW4xLmRhdGEsIGFlcyhsc3RhdCwgbWVkdikpICsKICBnZW9tX3BvaW50KCkgKwogIHN0YXRfc21vb3RoKG1ldGhvZCA9IGxtLCBmb3JtdWxhID0geSB+IHBvbHkoeCw1KSkKYGBgCgoKCiMjIyBVc2luZyBsb2cgdHJhbnNmb3JtYXRpb24KCmBgYHtyfQptb2QubG9nIDwtIGxtKG1lZHYgfiBsb2cobHN0YXQpLCBkYXRhID0gdHJhaW4xLmRhdGEpCnN1bW1hcnkobW9kLmxvZykKcHJlZGljdGlvbi5sb2cgPC0gbW9kLmxvZyAlPiUgcHJlZGljdCh0ZXN0MS5kYXRhKQoKZGF0YS5mcmFtZSgKICBSTVNFID0gUk1TRShwcmVkaWN0aW9uLmxvZywgdGVzdDEuZGF0YSRtZWR2KSwKICBSMiA9IFIyKHByZWRpY3Rpb24ubG9nLCB0ZXN0MS5kYXRhJG1lZHYpCikKYGBgCgpgYGB7cn0KZ2dwbG90KGRhdGEgPSB0cmFpbjEuZGF0YSwgYWVzKGxzdGF0LCBtZWR2KSkrCiAgZ2VvbV9wb2ludCgpICsKICBzdGF0X3Ntb290aChtZXRob2QgPSBsbSwgZm9ybXVsYSA9IHl+bG9nKHgpKQpgYGAKCiMjIyBTcGxpbmUgUmVncmVzc2lvbgoKYGBge3J9CmxpYnJhcnkoc3BsaW5lcykKa25vdHMgPC0gcXVhbnRpbGUodHJhaW4xLmRhdGEkbHN0YXQsIHAgPSBjKDAuMjUsIDAuNSwgMC43NSkpCm1vZGVsLnNwbGluZSA8LSBsbShtZWR2IH5icyhsc3RhdCxrbm90cz1rbm90cyksIGRhdGEgPSB0cmFpbjEuZGF0YSkKc3VtbWFyeShtb2RlbC5zcGxpbmUpCnByZWRpY3Rpb24uc3BsaW5lIDwtIG1vZGVsLnNwbGluZSAlPiUgcHJlZGljdCh0ZXN0MS5kYXRhKQoKZGF0YS5mcmFtZSgKICBSTVNFID0gUk1TRShwcmVkaWN0aW9uLnNwbGluZSwgdGVzdDEuZGF0YSRtZWR2KSwKICBSMiA9IFIyKHByZWRpY3Rpb24uc3BsaW5lLCB0ZXN0MS5kYXRhJG1lZHYpCikKYGBgCgoKCmBgYHtyfQpnZ3Bsb3QodHJhaW4xLmRhdGEsIGFlcyhsc3RhdCwgbWVkdikpICsgCiAgZ2VvbV9wb2ludCgpICsgCiAgc3RhdF9zbW9vdGgobWV0aG9kID0gbG0sIGZvcm11bGEgPSB5fmJzKHgsZGY9MykpCmBgYAoKCiMjIyBHZW5lcmFsaXplZCBhZGRpdGl2ZSBtb2RlbHMKCmBgYHtyfQpsaWJyYXJ5KG1nY3YpCgptb2RlbC5hZGRpdGl2ZSA8LSBnYW0obWVkdiB+IHMobHN0YXQpLCBkYXRhID0gdHJhaW4xLmRhdGEpCnN1bW1hcnkobW9kZWwuYWRkaXRpdmUpCgpwcmVkaWN0aW9uLmFkZGl0aXZlIDwtIG1vZGVsLmFkZGl0aXZlICU+JSBwcmVkaWN0KHRlc3QxLmRhdGEpCgpkYXRhLmZyYW1lKAogIFJNU0UgPSBSTVNFKHByZWRpY3Rpb24uYWRkaXRpdmUsIHRlc3QxLmRhdGEkbWVkdiksCiAgUjIgPSBSMihwcmVkaWN0aW9uLmFkZGl0aXZlLCB0ZXN0MS5kYXRhJG1lZHYpCikKYGBgCgoKYGBge3J9CmdncGxvdCh0cmFpbjEuZGF0YSwgYWVzKGxzdGF0LCBtZWR2KSkgKwogIGdlb21fcG9pbnQoKSArCiAgc3RhdF9zbW9vdGgobWV0aG9kID0gZ2FtLCBmb3JtdWxhID0geSB+IHMoeCkpCmBgYAoKCiMjIFJlZ3Jlc3Npb24gTW9kZWwgQWNjdXJhY3kgTWV0cmljcwoKIyMjIE1vZGVsIHBlcmZvcm1hbmNlIG1ldHJpY3MKCkluIHJlZ3Jlc3Npb24gbW9kZWwsIHRoZSBtb3N0IGNvbW1vbmx5IGtub3duIGV2YWx1YXRpb24gbWV0cmljcyBpbmNsdWRlOgoKMS4gX19SLXNxdWFyZWQgKFIyKV9fLCB3aGljaCBpcyB0aGUgcHJvcG9ydGlvbiBvZiB2YXJpYXRpb24gaW4gdGhlIG91dGNvbWUgdGhhdCBpcyBleHBsYWluZWQgYnkgdGhlIHByZWRpY3RvciB2YXJpYWJsZXMuIEluIG11bHRpcGxlIHJlZ3Jlc3Npb24gbW9kZWxzLCBSMiBjb3JyZXNwb25kcyB0byB0aGUgc3F1YXJlZCBjb3JyZWxhdGlvbiBiZXR3ZWVuIHRoZSBvYnNlcnZlZCBvdXRjb21lIHZhbHVlcyBhbmQgdGhlIHByZWRpY3RlZCB2YWx1ZXMgYnkgdGhlIG1vZGVsLgpUaGUgSGlnZXIgdGhlIFItc3F1YXJlZCwgdGhlIGJldHRlciB0aGUgbW9kZWwuCgoyLiBfX1Jvb3QgTWVhbiBTcXVhcmVkIEVycm9yIChSTVNFKV9fLCB3aGljaCBtZWFzdXJlcyB0aGUgYXZlcmFnZSBlcnJvciBwZXJmb3JtZWQgYnkgdGhlIG1vZGVsIGluIHRoZSBwcmVkaWN0aW5nIHRoZSBvdXRjb21lIGZvciBhbiBvYnNlcnZhdGlvbi4gTWF0aGVtYXRpY2FsbHksIHRoZSBSTVNFIGlzIHRoZSBzcXVhcmUgcm9vdCBvZiB0aGUgbWVhbiBzcXVhcmVkIGVycm9yIChNU0UpLCB3aGljaCBpcyB0aGUgYXZlcmFnZSBzcXVhcmVkIGRpZmZlcmVuY2UgYmV0d2VlbiB0aGUgb2JzZXJ2ZWQgYWN0dWFsIG91dGNvbWUgdmFsdWVzIGFuZCB0aGUgdmFsdWVzIHByZWRpY3RlZCBieSB0aGUgbW9kZWwuIFNvLCBNU0UgPSBtZWFuKChvYnNlcnZlZHMgLSBwcmVkaWN0ZWRzKV4yKSBhbmQgUk1TRSA9IHNxcnQoTVNFKS4gVGhlIGxvd2VyIHRoZSBSTVNFLCB0aGUgYmV0dGVyIHRoZSBtb2RlbC4KCjMuIF9fUmVzaWR1YWwgU3RhbmRhcmQgRXJyb3IgKFJTRSlfXywgYWxzbyBrbm93biBhcyB0aGUgX21vZGVsIHNpZ21hXywgaXMgYSB2YXJpYW50IG9mIHRoZSBSTVNFIGFkanVzdGVkIGZvciB0aGUgbnVtYmVyIG9mIHByZWRpY3RvcnMgaW4gdGhlIG1vZGVsLiBUaGUgbG93ZXIgdGhlIFJTRSwgdGhlIGJldHRlciB0aGUgbW9kZWwuIEluIHByYWN0aWNlLCB0aGUgZGlmZmVyZW5jZSBiZXR3ZWVuIFJNU0UgYW5kIFJTRSBpcyB2ZXJ5IHNtYWxsLCBwYXJ0aWN1bGFybHkgZm9yIGxhcmdlIG11bHRpdmFyaWF0ZSBkYXRhLgoKNC4gX19NZWFuIEFic29sdXRlIEVycm9yIChNQUUpX18sIGxpa2UgdGhlIFJNU0UsIHRoZSBNQUUgbWVhc3VyZXMgdGhlIHByZWRpY3Rpb24gZXJyb3IuIE1hdGhlbWF0aWNhbGx5LCBpdCBpcyB0aGUgYXZlcmFnZSBhYnNvbHV0ZSBkaWZmZXJlbmNlIGJldHdlZW4gb2JzZXJlZCBhbmQgcHJlZGljdGVkIG91dC1jb21lcywgTUFFID0gbWVhbihhYnMob2JzZXJ2ZWRzIC0gcHJlZGljdGVkcykpIC4gTUFFIGlzIGxlc3Mgc2Vuc2l0aXZlIHRvIG91dGxpZXJzIGNvbXBhcmVkIHRvIFJNU0UuCgpUaGUgcHJvYmxlbSB3aXRoIHRoZSBhYm92ZSBtZXRyaWNzLCBpcyB0aGF0IHRoZXkgYXJlIHNlbnNpYmxlIHRvIHRoZSBpbmNsdXNpb24gb2YgYWRkaXRpb25hbCB2YXJpYWJsZXMgaW4gdGhlIG1vZGVsLCBldmVuIGlmIHRob3NlIHZhcmlhYmxlcyBkb24ndCBoYXZlIHNpZ25pZmljYW50IGNvbnRyaWJ1dGlvbiBpbiBleHBsYWluaW5nIHRoZSBvdXRjb21lLiBQdXQgaW4gb3RoZXIgd29yZHMsIGluY2x1ZGluZyBhZGRpdGlvbmFsIHZhcmlhYmxlcyBpbiB0aGUgbW9kZWwgd2lsbCBhbHdheXMgaW5jcmVhc2UgdGhlIFIyIGFuZCByZWR1Y2UgdGhlIFJNU0UuIFNvLCB3ZSBuZWVkIGEgbW9yZSByb2J1c3QgbWV0cmljIHRvIGd1aWRlIHRoZSBtb2RlbCBjaG9pY2UuCgpDb25jZXJuaW5nIFIyLCB0aGVyZSBpcyBhbiBhZGp1c3RlZCB2ZXJzaW9uLCBjYWxsZWQgQWRqdXN0ZWQgUi1zcXVhcmVkLCB3aGljaCBhZGp1c3RzIHRoZSBSMiBmb3IgaGF2aW5nIHRvbyBtYW55IHZhcmlhYmxlcyBpbiB0aGUgbW9kZWwuCgpBZGRpdGlvbmFsbHksIHRoZXJlIGFyZSBmb3VyIG90aGVyIGltcG9ydGFudCBtZXRyaWNzIC0gQUlDLCBBSUNjLCBCSUMgYW5kIE1hbGxvd3MgQ3AgLSB0aGEgYXJlIGNvbW1vbmx5IHVzZWQgZm9yIG1vZGVsIGV2YWx1YXRpb24gYW5kIHNlbGVjdGlvbi4gVGhlc2UgYXJlIGFuIHVuYmlhc2VkIGVzdGltYXRlIG9mIHRoZSBtb2RlbCBwcmVkaWN0aW9uIGVycm9yIE1TRS4gVGhlIGxvd2VyIHRoZXNlIG1ldHJpY3MsIGhlIGJldHRlciB0aGUgbW9kZWwuCgoKMS4gX19BSUNfXyBzdGFuZHMgZm9yIChBa2lrZSdzIEluZm9ybWF0aW9uIENyaXRlcmlhKSwgYSBtZXRyaWMgZGV2ZWxvcGVlZCBieSB0aGUgSmFwYW5lc2UgU3RhdGlzdGljaWFuLCBIaXJvdHVndSBBa2Fpa2UsIDE5NzAuIFRoZSBiYXNpYyBpZGVhIG9mIEFJQyBpcyB0byBwZW5hbGl6ZSB0aGUgaW5jbHVzaW9uIG9mIGFkZGl0aW9uYWwgdmFyaWFibGVzIHRvIGEgbW9kZWwuIEl0IGFkZHMgYSBwZW5hbHR5IHRoYXQgaW5jcmVhc2VzIHRoZSBlcnJvciB3aGVuIGluY2x1ZGluZyBhZGRpdGlvbmFsIHRlcm1zLiBUaGUgbG93d2VyIHRoZSBBSUMsIHRoZSBiZXR0ZXIgdGhlIG1vZGVsLgoKCjIuIF9fQUlDY19fIGlzIGEgdmVyc2lvbiBvZiBBSUMgY29ycmVjdGVkIGZvciBzbWFsbCBzYW1wbGUgc2l6ZXMuCgozLiBfX0JJQ19fIChvciBCYXllc2lhbiBpbmZvcm1hdGlvbiBjcml0ZXJpYSkgaXMgYSB2YXJpYW50IG9mIEFJQyB3aXRoIGEgc3Ryb25nIHBlbmFsdHkgZm9yIGluY2x1ZGluZyBhZGRpdGlvbmFsIHZhcmlhYmxlcyB0byB0aGUgbW9kZWwuCgo0LiBfX01hbGxvd3MgQ3BfXzogQXZhcmlhbnQgb2YgQUlDIGRldmVsb3BlZCBieSBDb2xpbiBNYWxsb3dzLgoKR2VuZXJhbGx5LCBtb3N0IGNvbW1vbmx5IHVzZWQgbWV0cmljcywgZm9yIG1lYXN1cmluZyByZWdyZXNzaW9uIG1vZGVsIHF1YWxpdHkgYW5kIGNvbXBhcmluZyBtb2RlbHMsIGFyZTogQWRqdXN0ZWQgUjIsIEFJQywgQklDIGFuZCBDcAoKCkluIHRoZSBmb2xsb3dpbmcgc2VjdGlvbnMsIHdlJ2xsIHNobyB5b3UgaG93IHRvIGNvbXB1dGUgdGhlc2UgYWJvdmUgbWVudGlvbmVkIG1ldHJpY3MuCgoKYGBge3J9CmxpYnJhcnkobW9kZWxyKQpsaWJyYXJ5KGJyb29tKQpgYGAKCmBgYHtyfQojIExvYWQgdGhlIGRhdGEKZGF0YSgic3dpc3MiKQoKIyBJbnNwZWN0IHRoZSBkYXRhCnNhbXBsZV9uKHN3aXNzLDMpCmBgYAoKCiMjIyBCdWlsZGluZyByZWdyZXNzaW9uIG1vZGVscwoKV2Ugc3RhcnQgYnkgY3JlYXRpbmcgdHdvIG1vZGVsczoKCjEuIE1vZGVsIDEsIGluY2x1ZGluZyBhbGwgcHJlZGljdG9ycwoyLiBNb2RlbCAyLCBpbmNsdWRpbmcgYWxsIHByZWRpY3RvcnMgZXhjZXB0IHRoZSB2YXJpYWJsZSBfRXhhbWluYXRpb25fCgoKYGBge3J9Cm1vZGVsLnN3aXNzLjEgPC0gbG0oRmVydGlsaXR5fi4sIGRhdGEgPSBzd2lzcykKc3VtbWFyeShtb2RlbC5zd2lzcy4xKQpgYGAKCmBgYHtyfQoKZGF0YS5mcmFtZSgKICBSMiA9IHJzcXVhcmUobW9kZWwuc3dpc3MuMSwgZGF0YSA9IHN3aXNzKSwKICBSTVNFID0gcm1zZShtb2RlbC5zd2lzcy4xLCBkYXRhID0gc3dpc3MpLAogIE1BRSA9IG1hZShtb2RlbC5zd2lzcy4xLCBkYXRhID0gc3dpc3MpLAogIEFJQyA9IEFJQyhtb2RlbC5zd2lzcy4xKSwKICBCSUMgPSBCSUMobW9kZWwuc3dpc3MuMSkKKQpgYGAKCgpgYGB7cn0KbW9kZWwuc3dpc3MuMiA8LSBsbShGZXJ0aWxpdHl+LiAtIEV4YW1pbmF0aW9uLCBkYXRhID0gc3dpc3MpCgpzdW1tYXJ5KG1vZGVsLnN3aXNzLjIpCmBgYAoKCmBgYHtyfQpkYXRhLmZyYW1lKAogIFIyID0gcnNxdWFyZShtb2RlbC5zd2lzcy4yLCBkYXRhID0gc3dpc3MpLAogIFJNU0UgPSBybXNlKG1vZGVsLnN3aXNzLjIsIGRhdGEgPSBzd2lzcyksCiAgTUFFID0gbWFlKG1vZGVsLnN3aXNzLjIsIGRhdGEgPSBzd2lzcyksCiAgQUlDID0gQUlDKG1vZGVsLnN3aXNzLjIpLAogIEJJQyA9IEJJQyhtb2RlbC5zd2lzcy4yKQopCmBgYAoKYGBge3J9CmdsYW5jZShtb2RlbC5zd2lzcy4xKQpnbGFuY2UobW9kZWwuc3dpc3MuMikKYGBgCgoKTWFudWFsIGNvbXB1dGF0aW9uIG9mIFIyLCBSTVNFIGFuZCBNQUUKCmBgYHtyfQpzd2lzcyAlPiUgYWRkX3ByZWRpY3Rpb25zKG1vZGVsLnN3aXNzLjEpICU+JSBzdW1tYXJpc2UoCiAgUjIgPSBjb3IoRmVydGlsaXR5LCBwcmVkKV4yLAogIE1TRSA9IG1lYW4oRmVydGlsaXR5IC0gcHJlZCleMiwKICBSTVNFID0gc3FydChNU0UpLAogIE1BRSA9IG1lYW4oYWJzKEZlcnRpbGl0eSAtIHByZWQpKQopCmBgYAoKCiogX19Db21wYXJpbmcgcmVncmVzc2lvbiBtb2RlbHMgcGVyZm9ybWFuY2VfXwoKYGBge3J9CmdsYW5jZShtb2RlbC5zd2lzcy4xKSAlPiUgc2VsZWN0KGFkai5yLnNxdWFyZWQsIHNpZ21hLCBBSUMsIEJJQywgcC52YWx1ZSkKYGBgCgoKYGBge3J9CmdsYW5jZShtb2RlbC5zd2lzcy4yKSAlPiUgc2VsZWN0KGFkai5yLnNxdWFyZWQsIHNpZ21hLCBBSUMsIEJJQywgcC52YWx1ZSkKYGBgCgpGcm9tIHRoZSBvdXRwdXQgYWJvdmUsIGl0IGNhbiBiZSBzZWVuIHRoYXQ6CgoxLiBUaGUgdHdvIG1vZGVscyBoYXZlIGV4YWN0bHkgdGhlIHNhbWVkICpBZGp1c3RlZCBSMiogKDAuNjcpLCBtZWFuaW5nIHRoYXQgdGhleSBhcmUgZXF1aXZhbGVudCBpbiBleHBsYWluaW5nIHRoZSBvdXRjb21lLCBoZXJlIGZlcnRpbGl0eSBzY29yZS4gQWRkaXRpb25hbGx5LCB0aGV5IGhhdmUgdGhlIHNhbWUgYW1vdW50IG9mICpyZXNpZHVhbCBzdGFuZGFyZCBlcnJvciogKFJTRSBvciBzaWdtYSA9IDcuMTcpLiBIb3dldmVyLCB0aGUgbW9kZWwgMiBpcyBtb3JlIHNpbXBsZSB0aGFuIG1vZGVsIDEgYmVjYXVzZSBpdCBpbmNvcnBvcmF0ZXMgIGxlc3MgdmFyaWFibGVzLiBBbGwgdGhpbmdzIGVxdWFsLCB0aGUgc2ltcGxlIG1vZGVsIGlzIGFsd2F5cyBiZXR0ZXIgaW4gc3RhdGlzdGljcy4KCjIuIFRoZSBBSUMgYW5kIHRoZSBCSUMgb2YgdGhlIG1vZGVsIDIgYXJlIGxvd2VyIHRoYW4gdGhvc2Ugb2YgdGhlIG1vZGVsMS4gSW4gbW9kZWwgY29tcGFyaXNvbiBzdHJhdGVnaWVzLCB0aGUgbW9kZWwgd2l0aCB0aGUgbG93ZXN0IEFJQyBhbmQgQklDIHNjb3JlIGlzIHByZWZlcnJlZC4KCjMuIEZpbmFsbHksIHRoZSBGLXN0YXRpc3RpYyBwLnZhbHVlIG9mIHRoZSBtb2RlbCAyIGlzIGxvd2VyIHRoYW4gdGhlIG9uZSBvZiB0aGUgbW9kZWwgMS4gVGhpcyBtZWFucyB0aGF0IHRoZSBtb2RlbCAyIGlzIHN0YXRpc3RpY2FsbHkgbW9yZSBzaWduaWZpY2FudCBjb21wYXJlZCB0byBtb2RlbCAxLCB3aGljaCBpcyBjb25zaXN0ZW50IHRvIHRoZSBhdm92ZSBjb25jbHVzaW9uLgoKCk5vdGUgdGhhdCwgdGhlIFJNU0UgYW5kIHRoZSBSU0UgYXJlIG1lYXN1cmVkIGluIHRoZSBzYW1lIHNjYWxlIGFzIHRoZSBvdXRjb21lIHZhcmlhYmxlLgpEaXZpZGluZyB0aGUgUlNFIGJ5IHRoZSBhdmVyYWdlIHZhbHVlIG9mIHRoZSBvdXRjb21lIHZhcmlhYmxlIHdpbGwgZ2l2ZSB5b3UgdGhlIHByZWRpY3Rpb24gZXJyb3IgcmF0ZSwgd2hpY2ggc2hvdWxkIGJlIGFzIHNtYWxsIGFzIHBvc3NpYmxlOgoKYGBge3J9CnNpZ21hKG1vZGVsLnN3aXNzLjEpL21lYW4oc3dpc3MkRmVydGlsaXR5KQpgYGAKCkluIG91ciBleGFtcGxlIHRoZSBhdmVyYWdlIHByZWRpY3Rpb24gZXJyb3IgcmF0ZSBpcyAxMCUKCiMjIENyb3NzLXZhbGlkYXRpb24KCioqQ3Jvc3MtdmFsaWRhdGlvbioqIHJlZmVycyB0byBhIHNldCBvZiBtZXRob2RzIGZvciBtZWFzdXJpbmcgdGhlIHBlcmZvcm1hbmNlIG9mIGEgZ2l2ZW4gcHJlZGljdGl2ZSBtb2RlbCBvbiBuZXcgdGVzdCBkYXRhIHNldHMuCgpUaGUgYmFzaWMgaWRlYSwgYmVoaW5kIGNyb3NzLXZhbGlkYXRpb24gdGVjaG5pcXVlcywgY29uc2lzdHMgb2YgZGl2aWRpbmcgdGhlIGRhdGEgaW50byB0d28gc2V0czoKCjEuIFRoZSB0cmFpbmluZyBzZXQsIHVzZWQgdG8gdHJhaW4gKGkuZS4gYnVpbGQpIHRoZSBtb2RlbDsKMi4gYW5kIHRoZSB0ZXN0aW5nIHNldCAob3IgdmFsaWRhdGlvbiBzZXQpLCB1c2VkIHRvIHRlc3QgKGkuZS4gdmFsaWRhdGUpIHRoZSBtb2RlbCBieSBlc3RpbWF0aW5nIHRoZSBwcmVkaWN0aW9uIGVycm9yLgoKQ3Jvc3MtdmFsaWRhdGlvbiBpcyBhbHNvIGtub3duIGFzIGEgX3Jlc2FtcGxpbmcgbWV0aG9kXyBiZWNhdXNlIGl0IGludm9sdmVzIGZpdHRpbmcgdGhlIHNhbWUgc3RhdGlzdGljYWwgbWV0aG9kIG11bHRpcGxlIHRpbWVzIHVzaW5nIGRpZmZlcmVudCBzdWJzZXRzIG9mIHRoZSBkYXRhLgoKCjEuIFRoZSBkaWZmZXJlbnQgY3Jvc3MtdmFsaWRhdGlvbiBtZXRob2RzIGZvciBhc3Nlc3NpbmcgbW9kZWwgcGVyZm9ybWFuY2UuIFdlIGNvdmVyIHRoZSBmb2xsb3dpbmcgYXBwcm9jaGVzOgoKICAgICogdmFsaWRhdGlvbiBzZXQgYXBwcm9hY2ggKG9yIGRhdGEgc3BsaXQpCiAgICAqIExlYXZlIE9uIE91dCBDcm9zcyBWYWxpZGF0aW9uCiAgICAqIGstZm9sZCBDcm9zcyBWYWxpZGF0aW9uCiAgICAqIFJlcGVhdGVkIGstZm9sZCBDcm9zcyBWYWxpZGF0aW9uCiAKCiMjIE1vZGVsIHBlcmZvcm1hbmNlIG1ldHJpY3MKCkFmdGVyIGJ1aWxkaW5nIGEgbW9kZWwsIHdlIGFyZSBpbnRlcmVzdGVkIGluIGRldGVybWluaW5nIHRoZSBhY2N1cmFjeSBvZiB0aGlzIG1vZGVsIG9uIHByZWRpY3RpbmcgdGhlIG91dGNvbWUgZm9yIG5ldyB1bnNlZW4gb2JzZXJ2YXRpb25zIG5vdCB1c2VkIHRvIGJ1aWxkIHRoZSBtb2RlbC4gUHV0IGluIG90aGVyIHdvcmRzLCB3ZSB3YW50IHRvIGVzdGltYXRlIHRoZSBwcmVkaWN0aW9uIGVycm9yLgoKVG8gZG8gc28sIHRoZSBiYXNpYyBzdHJhdGVneSBpcyB0bzoKCjEuIEJ1aWxkIHRoZSBtb2RlbCBvbiBhIHRyYWluaW5nIGRhdGEgc2V0CjIuIEFwcGx5IHRoZSBtb2RlbCBvbiBhIG5ldyB0ZXN0IGRhdGEgc2V0IHRvIG1ha2UgcHJlZGljdGlvbnMKMy4gQ29tcHV0ZSB0aGUgcHJlZGljdGlvbiBlcnJvcnMKCgojIyMgQ3Jvc3MtdmFsaWRhdGlvbiBtZXRob2RzCgpCcmllZmx5LCBjcm9zcy12YWxpZGF0aW9uIGFsZ29yaXRobXMgY2FuIGJlIHN1bW1hcml6ZWQgYXMgZm9sbG93OgoKMS4gUmVzZXJ2ZSBhIHNtYWxsIHNhbXBsZSBvZiB0aGUgZGF0YSBzZXQKMi4gQnVpbGQgKG9yIHRyYWluKSB0aGUgbW9kZWwgdXNpbmcgdGhlIHJlbWFpbmluZyBwYXJ0IG9mIHRoZSBkYXRhIHNldCAKMy4gVGVzdCB0aGUgZWZmZWN0aXZlbmVzcyBvZiB0aGUgbW9kZWwgb24gdGhlIHJlc2VydmVkIHNhbXBsZSBvZiB0aGUgZGF0YSBzZXQuIElmIHRoZSBtb2RlbCB3b3JrcyB3ZWxsIG9uIHRoZSB0ZXN0IGRhdGEgc2V0LCB0aGVuIGl0J3MgZ29vZC4KClRoZSBmb2xsb3dpbmcgc2VjdGlvbnMgZGVzY3JpYmUgdGhlIGRpZmZmZXJlbnQgY3Jvc3MtdmFsaWRhdGlvbiB0ZWNobmlxdWVzLgoKCiMjIyMgMS4gVGhlIFZhbGlkYXRpb24gc2V0IEFwcHJvYWNoCgpUaGUgdmFsaWRhdGlvbiBzZXQgYXBwcm9hY2ggY29uc2lzdHMgb2YgcmFuZG9tbHkgc3BsaXR0aW5nIHRoZSBkYXRhIGludG8gdHdvIHNldHM6IG9uZSBzZXQgaXMgdXNlZCB0byB0cmFpbiB0aGUgbW9kZWwgYW5kIHRoZSByZW1haW5pbmcgb3RoZXIgc2V0IGlzIHVzZWQgdG8gdGVzdCB0aGUgbW9kZWwuCgpUaGUgcHJvY2VzcyB3b3JrcyBhcyBmb2xsb3c6CgoxLiBCdWlsZCAodHJhaW4pIHRoZSBtb2RlbCBvbiB0aGUgdHJhaW5pbmcgZGF0YSBzZXQKMi4gQXBwcGx5IHRoZSBtb2RlbCB0byB0aGUgdGVzdCBkYXRhIHNldCB0byBwcmVkaWN0IHRoZSBvdXRjb21lIG9mIG5ldyB1bnNlZW4gb2JzZXJ2YXRpb25zCjMuIFF1YW50aWZ5IHRoZSBwcmVkaWN0aW9uIGVycm9yIGFzIHRoZSBtZWFuIHNxdWFyZWQgZGlmZmVyZW5jZSBiZXR3ZWVuIHRoZSBvYnNlcnZlZCBhbmQgdGhlIHByZWRpY3RlZCBvdXRjb21lIHZhbHVlcy4KClRoZSBleGFtcGxlIGJlbG93IHNwbGl0cyB0aGUgc3dpc3MgZGF0YSBzZXQgc28gdGhhdCA4MCUgaXMgdXNlZCBmb3IgdHJhaW5pbmcgYSBsaW5lYXIgcmVncmVzc2lvbiBtb2RlbCBhbmQgMjAlIGlzIHVzZWQgdG8gZXZhbHVhdGUgdGhlIG1vZGVsIHBlcmZvcm1hbmNlLgoKYGBge3J9CiMgU3BsaXQgdGhlIGRhdGEgaW50byB0cmFpbmluZyBhbmQgdGVzdCBzZXQgCnNldC5zZWVkKDEyMykKCnRyYWluaW5nLnNhbXBsZXMgPC0gc3dpc3MkRmVydGlsaXR5ICU+JQogIGNyZWF0ZURhdGFQYXJ0aXRpb24ocD0wLjgsIGxpc3QgPSBGQUxTRSkKCnRyYWluLnN3aXNzLmRhdGEgPC0gc3dpc3NbdHJhaW5pbmcuc2FtcGxlcywgXQp0ZXN0LnN3aXNzLmRhdGEgPC0gc3dpc3NbLXRyYWluaW5nLnNhbXBsZXMsIF0KCiMgQnVpbGQgdGhlIG1vZGVsCgptb2RlbC5zd2lzcy5sbSA8LSBsbShGZXJ0aWxpdHkgfiAuLCBkYXRhID0gdHJhaW4uc3dpc3MuZGF0YSkKCiMgTWFrZSBwcmVkaWN0aW9ucyBhbmQgY29tcHV0ZSB0aGUgUjIsIFJNU0UgYW5kIE1BRQpwcmVkaWN0aW9ucy5zd2lzcy5sbSA8LSBtb2RlbC5zd2lzcy5sbSAlPiUgIHByZWRpY3QodGVzdC5zd2lzcy5kYXRhKQoKZGF0YS5mcmFtZSgKICBSMiA9IFIyKHByZWRpY3Rpb25zLnN3aXNzLmxtLCB0ZXN0LnN3aXNzLmRhdGEkRmVydGlsaXR5KSwKICBSTVNFID0gUk1TRShwcmVkaWN0aW9ucy5zd2lzcy5sbSwgdGVzdC5zd2lzcy5kYXRhJEZlcnRpbGl0eSksCiAgTUFFID0gTUFFKHByZWRpY3Rpb25zLnN3aXNzLmxtLCB0ZXN0LnN3aXNzLmRhdGEkRmVydGlsaXR5KQopCmBgYAoKCldoZW4gY29tcGFyaW5nIHR3byBtb2RlbHMsIHRoZSBvbmUgdGhhdCBwcm9kdWNlcyB0aGUgbG93ZXN0IHRlc3Qgc2FtcGxlIFJNU0UgaXMgdGhlIHByZWZlcnJlZCBtb2RlbC4KClRoZSBSTVNFIGFuZCB0aGUgTUFFIGFyZSBtZWFzdXJlZCBpbiB0aGUgc2FtZSBzY2FsZSBhcyB0aGUgb3V0Y29tZSB2YXJpYWJsZS4gRGl2aWRpbmcgdGhlIFJNU0UgYnkgdGhlIGF2ZXJhZ2UgdmFsdWUgb2YgdGhlIG91dGNvbWUgdmFyaWFibGUgd2lsbCBnaXZlIHlvdSB0aGUgcHJlZGljdGlvbiBlcnJvciByYXRlLCB3aGljaCBzb3VsZCBiZSBhcyBzbWFsbCBhcyBwb3NzaWJsZToKCmBgYHtyfQpSTVNFKHByZWRpY3Rpb25zLnN3aXNzLmxtLCB0ZXN0LnN3aXNzLmRhdGEkRmVydGlsaXR5KS9tZWFuKHRlc3Quc3dpc3MuZGF0YSRGZXJ0aWxpdHkpCmBgYAoKTm90ZSB0aGF0LCB0aGUgdmFsaWRhdGlvbiBzZXQgbWV0aG9kIGlzIG9ubHkgdXNlZnVsIHdoZW4geW91IGhhdmUgYSBsYXJnZSBkYXRhIHNldCB0aGF0IGNhbiBiZSBwYXJ0aXRpb25lZC4gQSBkaXNhZHZhbnRhZ2UgaXMgdGhhdCB3ZSBidWlsZCBhIG1vZGVsIG9uIGEgZnJhY3Rpb24gb2YgdGhlIGRhdGEgc2V0IG9ubHksIHBvc3NpYmx5IGxlYXZpbmcgb3V0IHNvbWUgaW50ZXJlc3RpbmcgaW5mb3JtYXRpb24gYWJvdXQgZGF0YSwgbGVhZGluZyB0byBoaWdoZXIgYmlhcy4gVGhlcmVmb3JlLCB0aGUgdGVzdCBlcnJvciByYXRlIGNhbiBiZSBoaWdobHkgdmFyaWFibGUsIGRlcGVuZGluZyBvbiB3aGljaCBvYnNlcnZhdGlvbnMgYXJlIGluY2x1ZGVkIGluIHRoZSB0cmFpbmluZyBzZXQgYW5kIHdoaWNoIG9ic2VydmF0aW9uIGFyZSBpbmNsdWRlZCBpbiB0aGUgdmFsaWRhdGlvbiBzZXQuCgoKIyMjIyAyLiBMZWF2ZSBvbmUgb3V0IGNyb3NzIHZhbGlkYXRpb24gLSBMT09DVgoKVGhpcyBtZXRob2Qgd29ya3MgYXMgZm9sbG93czoKMS4gTGVhdmUgb3V0IG9uZSBkYXRhIHBvaW50IGFuZCBidWlsZCB0aGUgbW9kZWwgb24gdGhlIHJlc3Qgb2YgdGhlIGRhdGEgc2V0CjIuIFRlc3QgdGhlIG1vZGVsIGFnYWluc3QgdGhlIGRhdGEgcG9pbnQgdGhhdCBpcyBsZWZ0IG91dCBhdCBzdGVwIDEgYW5kIHJlY29yZCB0aGUgdGVzdCBlcnJvciBhc3NvY2lhdGVkIHdpdGggdGhlIHByZWRpY3Rpb24KMy4gUmVwZWF0IHRoZSBwcm9jZXNzIGZvciBhbGwgZGF0YSBwb2ludHMKNC4gQ29tcG91dGUgdGhlIG92ZXJhbGwgcHJlZGljdGlvbiBlcnJvciBieSB0YWtpbmcgdGhlIGF2ZXJhZ2Ugb2YgYWxsIHRoZXNlIHRlc3QgZXJyb3IgZXN0aW1hdGVzIHJlY29yZWRlZCBhdCBzdGVwMy4KClByYWN0aWNhbCBleGFtcGxlIGluIFIgdXNpbmcgdGhlIGNhcmV0IHBhY2thZ2U6CgoKYGBge3J9CiMgRGVmaW5lIHRyYWluaW5nIGNvbnRyb2wKdHJhaW4uY29udHJvbCA8LSB0cmFpbkNvbnRyb2wobWV0aG9kID0gIkxPT0NWIikKCiMgVHJhaW4gdGhlIG1vZGVsCgptb2RlbC5sb29jdiA8LSB0cmFpbihGZXJ0aWxpdHkgfi4sIGRhdGEgPSBzd2lzcywgbWV0aG9kID0gImxtIiwgdHJDb250cm9sID0gdHJhaW4uY29udHJvbCkKIyBTdW1tYXJpemUgdGhlIHJlc3VsdHMKcHJpbnQobW9kZWwubG9vY3YpCmBgYAoKClRoZSBhZHZhbnRhZ2Ugb2YgdGhlIExPT0NWIG1ldGhvZCBpcyB0aGF0IHdlIG1ha2UgdXNlIGFsbCBkYXRhIHBvaW50cyByZWR1Y2luZyBwb3RlbnRpYWwgYmlhcy4KCkhvd2V2ZXIsIHRoZSBwcm9jZXNzIGlzIHJlcGVhdGVkIGFzIG1hbnkgdGltZXMgYXMgdGhlcmUgYXJlIGRhdGEgcG9pbnRzLCByZXN1bHRpbmcgdG8gYSBoaWdoZXIgZXhlY3Rpb24gdGltZSB3aGVuIG4gaXMgZXh0cmVhbWVseSBsYXJnZS4KCkFkZGl0aW9uYWxseSwgd2UgdGVzdCB0aGUgbW9kZWwgcGVyZm9ybWFuY2UgYWdhaW5zdCBvbmUgZGF0YSwgcG9pbnQgYXQgZWFjaCBpdGVyYXRpb24uIFRoaXMgbWlnaHQgcmVzdWx0IHRvIGhpZ2hlciB2YXJpYXRpb24gaW4gdGhlIHByZWRpY3Rpb24gZXJyb3IsIGlmIHNvbWUgZGF0YSBwb2ludHMgYXJlIG91dGxpZXJzLiBTbywgd2UgbmVlZCBhIGdvb2QgcmF0aW8gb2YgdGVzdGluZyBkYXRhIHBvaW50cywgYSBzb2x1dGlvbiBwcm92aWRlZCBieSB0aGUgKiprLWZvbGQgY3Jvc3MtdmFsaWRhdGlvbiBtZXRob2QqKi4KCiMjIyMgSy1mb2xkIGNyb3NzLXZhbGlkYXRpb24KClRoZSBrLWZvbGQgY3Jvc3MgdmFsaWRhdGlvbiBtZXRob2QgZXZhbHVhdGVzIHRoZSBtb2RlbCBwZXJmb3JtYW5jZSBvbiBkaWZmZXJlbnQgc3Vic2V0IG9mIHRoZSB0cmFpbmluZyBkYXRhIGFuZCB0aGVuIGNhbGN1bGF0ZSB0aGUgYXZlcmFnZSBwcmVkaWN0aW9uIGVycm9yIHJhdGUuIFRoZSBhbGdvcml0aG0gaXMgYXMgZm9sbG93OgoKMS4gUmFuZG9tbHkgc3BsaXQgdGhlIGRhdGEgaW50byBrLXN1YnNldCAob3Igay1mb2xkKSAoZm9yIGV4YW1wbGUgNSBzdWJzZXRzKQoyLiBSZXNlcnZlIG9uIHN1YnNldCBhbmQgdHJhaW4gdGhlIG1vZGVsIG9uIGFsbCBvdGhlciBzdWJzZXRzCjMuIFRlc3QgdGhlIG1vZGVsIG9uIHRoZSByZXNlcnZlZCBzdWJzZXQgYW5kIHJlY29yZCB0aGUgcHJlZGljdGlvbiBlcnJvcgo0LiBSZXBlYXQgdGhpcyBwcm9jc3MgdW50aWxsIGVhY2ggb2YgdGhlIGsgc3Vic2V0cyBoYXMgc2VydmVkIGFzIHRoZSB0ZXN0IHNldC4KNS4gQ29tcHV0ZSB0aGUgYXZlcmFnZSBvZiB0aGUgayByZWNvcmRlZCBlcnJvcnMuIFRoaXMgaXMgY2FsbGVkIHRoZSBjcm9zcyB2YWxpZGF0aW9uIGVycm9yIHNlcnZpbmcgYXMgdGhlIHBlcmZvcm1hbmNlIG1ldHJpYyBmb3IgdGhlIG1vZGVsCgpLLWZvbGQgY3Jvc3MtdmFsaWRhdGlvbiAoQ1YpIGlzIGEgcm9idXN0IG1ldGhvZCBmb3IgZXN0aW1hdGluZyB0aGUgYWNjdXJhY3kgb2YgYSBtb2RlbC4KVGhlIG1vc3Qgb2J2aW91cyBhZHZhbnRhZ2Ugb2Ygay1mb2xkIENWIGNvbXBhcmVkIHRvIExPT0NWIGlzIGNvbXB1dGF0aW9uYWwuIEEgbGVzcyBvYnZpb3VzIGJ1dCBwb3RlbnRpYWxseSBtb3JlIGltcG9ydGFudCBhZHZhbnRhZ2Ugb2Ygay1mb2xkIENWIGlzIHRoYXQgaXQgb2Z0ZW4gZml2ZXMgbW9yZSBhY2N1cmF0ZSBlc3RpbWF0ZXMgb2YgdGhlIHRlc3QgZXJyb3IgcmF0ZSB0aGFuIGRvZXMgTE9PQ1YKCgo+IEluIHByYWN0aWNlLCBvbmUgdHlwaWNhbGx5IHBlcmZvcm1zIGstZm9sZCBjcm9zcy12YWxpZGF0aW9uIHVzaW5nIGsgPSA1IG9yIGsgPSAxMCwgYXMgdGhlc2UgdmFsdWVzIGhhdmUgYmVlbiBzaG93biBlbXBpcmljYWxseSB0byB5aWVsZCB0ZXN0IGVycm9yIHJhdGUgZXN0aW1hdGVzIHRoYXQgc3VmZmVyIG5laXRoZXIgZnJvbSBleGNlc3NpdmVseSBoaWdoIGJpYXMgbm9yIGZyb20gdmVyeSBoaWdoIHZhcmlhbmNlLgoKVGhlIGZvbGxvd2luZyBleGFtcGxlIHVzZXMgMTAtZm9sZCBjcm9zcyB2YWxpZGF0aW9uIHRvIGVzdGltYXRlIHRoZSBwcmVkaWN0aW9uIGVycm9yLiBNYWtlIHN1cmUgdG8gc2V0IHNlZWQgZm9yIHJlcHJvZHVjaWJpbGl0eS4KCgpgYGB7cn0KIyBEZWZpbmUgdHJhaW5pbmcgY29udHJvbApzZXQuc2VlZCgxMjMpCgp0cmFpbi5jb250cm9sIDwtIHRyYWluQ29udHJvbChtZXRob2QgPSAiY3YiLCBudW1iZXIgPSAxMCkKCiMgVHJhaW4gdGhlIG1vZGVsCm1vZGVsLmN2IDwtIHRyYWluKEZlcnRpbGl0eX4uLCBkYXRhID0gc3dpc3MsIG1ldGhvZCA9ICJsbSIsIHRyQ29udHJvbCA9IHRyYWluLmNvbnRyb2wpCgojIFN1bW1hcml6ZSB0aGUgcmVzdWx0cwpwcmludChtb2RlbC5jdikKYGBgCgojIyMjIDQuIFJlcGVhdGVkIEstZm9sZCBjcm9zcy12YWxpZGF0aW9uCgpUaGUgcHJvY2VzcyBvZiBzcGxpdHRpbmcgdGhlIGRhdGEgaW50byBrLWZvbGRzIGNhbiBiZSByZXBlYXRlZCBhIG51bWJlciBvZiB0aW1lcywgdGhpcyBpcyBjYWxsZWQgcmVwZWF0ZWQgay1mb2xkIGNyb3NzIHZhbGlkYXRpb24uCgoKIyMjIEJvb3RzdHJhcCBwcm9jZWR1cmUKClRoZSBib290c3RyYXAgbWV0aG9kIGlzIHVzZWQgdG8gcXVhbnRpZnkgdGhlIHVuY2VydGFpbnR5IHdpdGggYSBnaXZlbiBzdGF0aXN0aWNhbCBlc3RpbWF0b3Igb3Igd2l0aCBhIHByZWRpY3RpdmUgbW9kZWwuCgpJdCBjb25zaXN0cyBvZiByYW5kb21seSBzZWxlY3RpbmcgYSBzYW1wbGUgb2YgbiBvYnNlcnZhdGlvbnMgZnJvbSB0aGUgb3JpZ2luYWwgZGF0YSBzZXQuIFRoaXMgc3Vic2V0LCBjYWxsZWQgYm9vdHN0cmFwIGRhdGEgc2V0IGlzIHRoZW4gdXNlZCB0byBldmFsdWF0ZSB0aGUgbW9kZWwuCgpUaGlzIHByb2NlZHVyZSBpcyByZXBlYXRlZCBhIGxhcmdlIG51bWJlciBvZiB0aW1lcyBhbmQgdGhlIHN0YW5kYXJkIGVycm9yIG9mIHRoZSBib290c3RyYXAgZXN0aW1hdGUgaXMgdGhlbiBjYWxjdWxhdGVkLiBUaGUgcmVzdWx0cyBwcm92aWRlIGFuIGluZGljYXRpb24gb2YgdGhlIHZhcmlhbmNlIG9mIHRoZSBtb2RlbCBwZXJmb3JtYW5jZS4KCk5vdGUgdGhhdCwgdGhlIHNhbXBsaW5nIGlzIHBlcmZvcm1lZCB3aXRoIHJlcGxhY2VtZW50LCB3aGljaCBtZWFucyB0aGF0IHRoZSBzYW1lIG9ic2VydmF0aW9uIGNhbiBvY2N1ciBtb3JlIHRoYW4gb25jZSBpbiB0aGUgYm9vc3RycmFwIGRhdGEgc2V0LgoKIyMjIyBFdmFsdWF0aW5nIGEgcHJlZGljdGl2ZSBtb2RlbCBwZXJmb3JtYW5jZQoKCmBgYHtyfQojIERlZmluZSB0cmFpbmluZyBjb250cm9sCgp0cmFpbi5jb250cm9sLmJvb3RzdHJhcCA8LSB0cmFpbkNvbnRyb2wobWV0aG9kID0gImJvb3QiLCBudW1iZXIgPSAxMDApCgptb2RlbC5ib290c3RyYXAgPC0gdHJhaW4oRmVydGlsaXR5IH4uLCBkYXRhID0gc3dpc3MsIG1ldGhvZCA9ICJsbSIsIHRyQ29udHJvbCA9IHRyYWluLmNvbnRyb2wuYm9vdHN0cmFwKQoKIyBTdW1tYXJpemUgdGhlIHJlc3VsdHMKcHJpbnQobW9kZWwuYm9vdHN0cmFwKQpgYGAKCgojIyMgUXVhbnRpZnlpbmcgYW4gZXN0aW1hdG9yIHVuY2VydGFpbnR5IGFuZCBjb25maWRlbmNlIGludGVydmFscwoKVGhlIGJvb3RzdHJhcCBhcHByb2FjaCBjYW4gYmUgdXNlZCB0byBxdWFudGlmeSB0aGUgdW5jZXJ0YWludHkgKG9yIHN0YW5kYXJkIGVycm9yKSBhc3NvY2lhdGVkIHdpdGggYW55IGdpdmVuIHN0YXRpc3RpY2FsIGVzdGltYXRvci4KCkZvciBleGFtcGxlLCB5b3UgbWlnaHQgd2FudCB0byBlc3RpbWF0ZSB0aGUgYWNjdXJhY3kgb2YgdGhlIGxpbmVhciByZWdyZXNzaW9uIGJldGEgY29lZmZpY2llbnRzIHVzaW5nIGJvb3RzdHJhcCBtZXRob2QuCgpUaGUgZGlmZmVyZW50IHN0ZXBzIGFyZSBhcyBmb2xsb3dzOgoKMS4gQ3JlYXRlIGEgc2ltcGxlIGZ1bmN0aW9uLCBtb2RlbF9jb2VmKCksIHRoYXQgdGFrZXMgdGhlICoqc3dpc3MqKiBkYXRhIHNldCBhcyB3ZWxsIGFzIHRoZSBpbmRpY2VzIGZvciB0aGUgb2JzZXJ2YXRpb25zLCBhbmQgcmV0dXJucyB0aGUgcmVncmVzc2lvbiBjb2VmZmljaWVudHMuCjIuIEFwcGx5IHRoZSBmdW5jdGlvbiBib290X2Z1bigpIHRvIHRoZSBmdWxsIGRhdGEgc2V0IG9mIDQ3IG9ic2VydmF0aW9ucyBpbiBvcmRlciB0byBjb21wdXRlIHRoZSBjb2VmZmljaWVudHMKCldlIHN0YXJ0IGJ5IGNyZWF0aW5nIGEgZnVuY3Rpb24gdGhhdCByZXRybnMgdGhlIHJlZ3Jlc3Npb24gbW9kZWwgY29lZmZpY2VpZW50czoKCgpgYGB7cn0KbW9kZWxfY29lZiA8LSBmdW5jdGlvbihkYXRhLCBpbmRleCkgewogIGNvZWYobG0oRmVydGlsaXR5fi4sIGRhdGEgPSBkYXRhLCBzdWJzZXQgPSBpbmRleCkpCn0KCm1vZGVsX2NvZWYoc3dpc3MsIDE6NDcpCmBgYAoKCmBgYHtyfQpsaWJyYXJ5KGJvb3QpCmJvb3Qoc3dpc3MsIG1vZGVsX2NvZWYsIDUwMCkKYGBgCgpJbiB0aGUgb3V0cHV0IGFib3ZlLAoKKiAqKm9yaWdpbmFsKiogY29sdW1uIGNvcnJlc3BvbmRzIHRvIHRoZSByZWdyZXNzaW9uIGNvZWZmaWNpZW50cy4gVGhlIGFzc29jaWF0ZWQgc3RhbmRhcmQgZXJyb3JzIGFyZSBnaXZlbiBpbiB0aGUgY29sdW1uICoqc3RkLmVycm9yKiogLgoKKiAqKnQxKiogY29ycmVzcG9uZHMgdG8gdGhlIGludGVyY2VwdCwgKip0MioqIGNvbnJyZXNwb25kcyB0byAqQWdyaWN1bHR1cmUqKiBhbmQgc28gb24gLi4KCkZvciBleGFtcGxlLCBpdCBjYW4gYmUgb2JzZXJ2ZSB0aGF0LCB0aGUgc3RhbmRhcmQgZXJyb3IgKFNFKSBvZiB0aGUgcmVncmVzc2lvbiBjb2VmZmljaWVudCBhc3NvY2lhdGVkIHdpdGggKipBZ3JpY3VsdHVyZSoqIGlzIDAuMDYKCk5vdGUgdGhhdCwgdGhlIHN0YW5kYXJkIGVycm9ycyBtZWFzdXJlIHRoZSB2YXJpYWJpbGl0eS9hY2N1cmFjeSBvZiB0aGUgYmV0YSBjb2VmZmljaWVudHMuIEl0IGNhbiBiZSB1c2VkIHRvIGNvbXB1dGUgdGhlIGNvbmZpZGVuY2UgaW50ZXJ2YWxzIG9mIHRoZSBjb2VmZmljaWVudHMuCgpGb3IgZXhhbXBsZSwgdGhlIDk1JSBjb25maWRlbmNlIGludGVydmFsIGZvciBhIGdpdmVuIGNvZWZmaWNpZW50IGIgaXMgZGVmaW5lZCBhcyAqKmIrLy0xLjk2XCpTRShiKSoqLAp3aGVyZToKCiogVGhlIGxvd2VyIGxpbWl0cyBvZiBiID0gYi0xLjk2XCpTRShiKSA9IC0wLjE3MiAtICgxLjk2XCowLjA2ODApID0gLTAuMzA2KGZvciBBZ3JpY3VsdHVyZSB2YXJpYWJsZSkKKiBUaGUgVXBwZXIgbGltaXRzIG9mIGIgPSBiKzEuOTZcKlNFKGIpID0gLTAuMTcyICsgKDEuOTZcKjAuMDY4MCkgPSAtMC4wMzc0MzEyMShmb3IgQWdyaWN1bHR1cmUgdmFyaWFibGUpCgpUaGF0IGlzLCB0aGVyZSBpcyBhcHByb3hpbWF0ZWx5IGEgOTUlIGNoYW5jZSB0aGF0IHRoZSBpbnRlcnZhbFstMC4zMDYsLTAuMDM2XSB3aWxsIGNvbnRhaW4gdGhlIHRydWUgdmFsdWUgb2YgdGhlIGNvZWZmaWNpZW50LgoKVXNpbmcgdGhlIHN0YW5kYXJkIGxtKCkgZnVuY3Rpb24gZ2l2ZXMgYSBzbGlnaHRseSBkaWZmZXJlbnQgc3RhbmRhcmQgZXJyb3JzLCBiZWNhdXNlIHRoZSBsaW5lYXIgbW9kZWwgbWFrZSBzb21lIGFzc3VtcHRpb25zIGFib3V0IHRoZSBkYXRhOgoKCmBgYHtyfQpzdW1tYXJ5KGxtKEZlcnRpbGl0eSB+LiwgZGF0YSA9IHN3aXNzKSkkY29lZgoKYGBgCgoKVGhlIGJvb3RzdHJhcCBhcHByb2FjaCBkb2VzIG5vdCByZWx5IG9uIGFueSBvZiB0aGVzZSBhc3N1bXB0aW9ucyBtYWRlIGJ5IHRoZSBsaW5lYXIgbW9kZWwgYW5kIHNvIGl0IGlzIGxpa2VseSBnaXZpbmcgYSBtb3JlIGFjY3VyYXRlIGVzdGltYXRlIG9uIHRoZSBjb2VmZmljaWVudHMgc3RhbmRhcmQgZXJyb3JzIHRoYW4gdGlzIHRoZSBzdW1tYXJ5KCkgZnVuY3Rpb24uCgoKIyMgTW9kZWwgU2VsZWN0aW9uCgojIyBCZXN0IFN1YnNldHMgUmVncmVzc2lvbgoKYGBge3J9CmxpYnJhcnkodGlkeXZlcnNlKQpsaWJyYXJ5KGNhcmV0KQpsaWJyYXJ5KGxlYXBzKQpgYGAKCgojIyMgQ29tcHV0aW5nIGJlc3Qgc3Vic2V0IHJlZ3Jlc3Npb24KClRoZSBSIGZ1bmN0aW9uIHJlZ3N1YnNldHMoKSBbbGVhcHMgcGFja2FnZV0gY2FuIGJlIHVzZWQgdG8gaWRlbnRpZnkgZGlmZmVyZW50IGJlc3QgbW9kZWxzIG9mIGRpZmZlcmVudCBzaXplcy4KCllvdSBuZWVkIHRvIHNwZWNpZnkgdGhlIG9wdGlvbiAqKm52bWF4KiosIHdoaWNoIHJlcHJlc2VudHMgdGhlIG1heGltdW0gbnVtYmVyIG9mIHByZWRpY3RvcnMgdG8gaW5jb3Jwb3JhdGUgaW4gdGhlIG1vZGUuIEZvciBleGFtcGxlLCBpZiBudm1heCA9IDUsIHRoZSBmdW5jdGlvbiB3aWxsIHJldHVybiB1cCB0byB0aGUgYmVzdCA1LXZhcmlhYmxlcyBtb2RlbCwgdGhhdCBpcywgaXQgcmV0dXJucyB0aGUgYmVzdCAxLXZhcmFibGUgbW9kZWwsIHRoZSBiZXN0IDItdmFyaWFibGVzIG1vZGVsLCAuLiwgdGhlIGJlc3QgNS12YXJpYWJsZXMgbW9kZWxzLgoKSW4gb3VyIGV4YW1wbGUsIHdlIGhhdmUgb25seSA1IHByZWRpY3RvciB2YXJpYWJsZXMgaW4gdGhlIGRhdGEuIFNvLCB3ZSdsbCB1c2UgbnZtYXggPSA1CgoKCgpgYGB7cn0KbW9kZWxzLnJlZ3N1YnNldHMgPC0gcmVnc3Vic2V0cyhGZXJ0aWxpdHl+LiwgZGF0YT0gc3dpc3MsIG52bWF4ID0gNSkKc3VtbWFyeShtb2RlbHMucmVnc3Vic2V0cykKYGBgCgoKIyMjIE1vZGVsIHNlbGVjdGlvbiBjcml0ZXJpYTogQWRqdXN0ZWQgUjIsIENwIGFuZCBCSUMKClRoZSAqKnN1bW1hcnkoKSoqIGZ1bmN0aW9uIHJldHVybnMgc29tZSBtZXRyaWNzIC0gQWRqdXN0ZWQgUjIsIENwIGFuZCBCSUMgYWxsb3dpbmcgdXMgdG8gaWRlbnRpZnkgdGhlIGJlc3Qgb3ZlcmFsbCBtb2RlbCwgd2hlcmUgYmVzdCBpcyBkZWZpbmVkIGFzIHRoZSBtb2RlbCB0aGF0IG1heGltaXplIHRoZSBhZGp1c3RlZCBSMiBhbmQgbWluaW1pemUgdGhlIHByZWRpY3Rpb24gZXJyb3JzCgpUaGUgYWRqdXN0ZWQgUjIgcmVwcmVzZW50cyB0aGUgcHJvcG9ydGlvbiBvZiB2YXJpYXRpb24sIGluIHRoZSBvdXRjb21lLCB0aGF0IGFyZSBleHBsYWluZWQgYnkgdmFyaWF0aW9uIGluIHByZWRpY3RvcnMgdmFsdWVzLiBUaGUgaGlnaGVyIHRoZSBhZGp1c3RlZCBSMiwgdGhlIGJldHRlciB0aGUgbW9kZWwuCgpUaGUgYmVzdCBtb2RlbCwgYWNjb3JkaW5nIHRvIGVhY2ggb2YgdGhlc2UgbWV0cmljcywgY2FuIGJlIGV4dHJhY3RlZCBhcyBmb2xsb3c6CgoKYGBge3J9CnJlcy5zdW0gPC0gc3VtbWFyeShtb2RlbHMucmVnc3Vic2V0cykKCmRhdGEuZnJhbWUoCiAgQWRqLlIyID0gd2hpY2gubWF4KHJlcy5zdW0kYWRqcjIpLAogIENQID0gd2hpY2gubWluKHJlcy5zdW0kY3ApLAogIEJJQyA9IHdoaWNoLm1pbihyZXMuc3VtJGJpYykKKQoKYGBgCgoKClRoZXJlIGlzIG5vIHNpbmdsZSBjb3JyZWN0IHNvbHV0aW9uIHRvIG1vZGVsIHNlbGVjdGlvbiwgZWFjaCBvZiB0aGVzZSBjcml0ZXJpYSB3aWxsIGxlYWQgdG8gc2xpZ2h0bHkgZGlmZmVyZW50IG1vZGVscy4KCgpSZW1lbWJlcnQgdGhhdCwgCgo+ImFsbCBtb2RlbHMgYXJlIHdyb25nLCBzb21lIG1vZGVscyBhcmUgdXNlZnVsIgoKU28sIHdlIGhhdmUgZGlmZmVyZW50ICJiZXN0IiBtb2RlbHMgZGVwZW5kaW5nIG9uIHdoaWNoIG1ldHJpY3Mgd2UgY29uc2lkZXIuIFdlIG5lZWQgYWRkaXRvbmFsIHN0cmF0ZWdpZXMuCgpOb3RlIGFsc28gdGhhdCB0aGUgYWRqdXN0ZWQgUjIsIEJJQyBhbmQgQ3AgYXJlIGNhbGN1bGF0ZWQgb24gdGhlIHRyYWluaW5nIGRhdGEgdGhhdCBoYXZlIGJlZW4gdXNlZCB0byBmaXQgdGhlIG1vZGVsLiBUaGlzIG1lYW5zIHRoYXQsIHRoZSBtb2RlbCBzZWxlY3Rpb24sIHVzaW5nIHRoZXNlIG1ldHJpY3MsIGlzIHBvc3NpYmx5IHN1YmplY3QgdG8gb3ZlcmZpdHRpbmcgYW5kIG1heSBub3QgcGVyZm9ybSBhcyB3ZWxsIHdoZW4gYXBwbGllZCB0byBuZXcgZGF0YS4KCkEgbW9yZSByaWdvcm91cyBhcHByb2FjaCBpcyB0byBzZWxlY3QgYSBtb2RlbHMgYmFzZWQgb24gdGhlIHByZWRpY3Rpb24gZXJyb3IgY29tcHV0ZWQgb24gYSBuZXcgdGVzdCBkYXRhIHVzaW4gay1mb2xkIGNyb3NzLXZhbGlkYXRpb24gdGVjaG5pcXVlcwoKIyMjIEstZm9sZCBjcm9zcy12YWxpZGF0aW9uIAoKSGVyZSwgd2UnbGwgZm9sbG93IHRoZSBwcm9jZWR1cmUgYmVsb3c6CgoxLiBFeHRyYWN0IHRoZSBkaWZmZXJlbnQgbW9kZWwgZm9ybXVsYXMgZnJvbSB0aGUgbW9kZWxzIG9iamVjdAoyLiBUcmFpbiBhIGxpbmVhciBtb2RlbCBvbiB0aGUgZm9ybXVsYSB1c2luZyBrLWZvbGQgY3Jvc3MtdmFsaWRhdGlvbih3aXRoIGs9NSkgYW5kIGNvbXB1dGUgdGhlIHByZWRpY3Rpb24gZXJyb3Igb2YgZWFjaCBtb2RlbAoKV2Ugc3RhcnQgYnkgZGVmaW5pbmcgdHdvIGhlbHBlciBmdW5jdGlvbjoKCjEuIGdldF9tb2RlbF9mb3JtdWxhKCksIGFsbG93aW5nIHRvIGFjY2VzcyBlYXNpbHkgdGhlIGZvcm11bGEgb2YgdGhlIG1vZGVscyByZXR1cm5lZCBieSB0aGUgZnVuY3Rpb24gKipyZWdzdWJzZXRzKCkqKi4KCgpgYGB7cn0KI2lkOiBtb2RlbCBpZAojb2JqZWN0OiByZWdzdWJzZXRzIG9iamVjdAojZGF0YTogZGF0YSB1c2VkIHRvIGZpdCByZWdzdWJzZXRzCgpnZXRfbW9kZWxfZm9ybXVsYSA8LSBmdW5jdGlvbihpZCwgb2JqZWN0KXsKICBtb2RlbHMgPC0gc3VtbWFyeShvYmplY3QpJHdoaWNoW2lkLC0xXQogICNnZXQgb3V0Y29tZSB2YXJpYWJsZQogIAogIGZvcm0gPC0gYXMuZm9ybXVsYShvYmplY3QkY2FsbFtbMl1dKQogIG91dGNvbWUgPC0gYWxsLnZhcnMoZm9ybSlbMV0KICAjR2V0IG1vZGVsIHByZWRpY3RvcnMKICBwcmVkaWN0b3JzIDwtIG5hbWVzKHdoaWNoKG1vZGVscyA9PSBUUlVFKSkKICBwcmVkaWN0b3JzIDwtIHBhc3RlKHByZWRpY3RvcnMsIGNvbGxhcHNlID0gIisiKQogICMgQnVpbGQgbW9kZWwgZm9ybXVsYQogIGFzLmZvcm11bGEocGFzdGUob3V0Y29tZSwgIn4iLCBwcmVkaWN0b3JzKSkKfQpgYGAKCgpgYGB7cn0KZ2V0X21vZGVsX2Zvcm11bGEoMyxtb2RlbHMucmVnc3Vic2V0cykKCmBgYAoKCjIuIGdldF9jdl9lcnJvcigpLCB0byBnZXQgdGhlIGNyb3NzLXZhbGlkYXRpb24gKENWKSBlcnJvciBmb3IgYSBnaXZlbiBtb2RlbDoKCmBgYHtyfQpnZXRfY3ZfZXJyb3IgPC0gZnVuY3Rpb24gKG1vZGVsLmZvcm11bGEsIGRhdGEpewogIHNldC5zZWVkKDEpCiAgIHRyYWluLmNvbnRyb2wgPC0gdHJhaW5Db250cm9sKG1ldGhvZCA9ICJjdiIsIG51bWJlciA9IDUpCiAgIGN2IDwtIHRyYWluKG1vZGVsLmZvcm11bGEsIGRhdGEgPSBkYXRhLCBtZXRob2QgPSAibG0iLCB0ckNvbnRyb2wgPSB0cmFpbi5jb250cm9sKQogICBjdiRyZXN1bHRzJFJNU0UKfQpgYGAKClVzZSB0aGUgYWJvdmUgZGVmaW5lZCBtZXRob2QgdG8gY29tcHV0ZSB0aGUgcHJlZGljdGlvbiBlcnJvcgoKCmBgYHtyfQojIENvbXB1dGUgY3Jvc3MtdmFsaWRhdGlvbiBlcnJvcgoKbW9kZWwuaWRzIDwtIDE6NQpjdi5lcnJvcnMgPC0gbWFwKG1vZGVsLmlkcywgZ2V0X21vZGVsX2Zvcm11bGEsIG1vZGVscy5yZWdzdWJzZXRzKSAlPiUKICBtYXAoZ2V0X2N2X2Vycm9yLCBkYXRhID0gc3dpc3MpICU+JQogIHVubGlzdCgpCmN2LmVycm9ycwoKYGBgCgpgYGB7cn0KIyBTZWxlY3QgdGhlIG1vZGVsIHRoYXQgbWluaW1pemUgdGhlIENWIGVycm9yCndoaWNoLm1pbihjdi5lcnJvcnMpCgpgYGAKCkl0IGNhbiBiZSBzZWVuIHRoYXQgdGhlIG1vZGVsIHdpdGggNCB2YXJpYWJsZXMgaXMgdGhlIGJlc3QgbW9kZWwuIEl0IGhhcyB0aGUgbG93ZXIgcHJlZGljdGlvbiBlcnJvci4gVGhlIHJlZ3Jlc3Npb24gY29lZmZpY2llbnRzIG9mIHRoaXMgbW9kZWwgY2FuIGJlIGV4dHJhY3RlZCBhcyBmb2xsb3c6CgpgYGB7cn0KY29lZihtb2RlbHMucmVnc3Vic2V0cywgNCkKYGBgCgoKIyMgU3RlcHdpc2UgUmVncmVzc2lvbgoKMS4gKipGb3J3YXJkIHNlbGVjdGlvbioqOiBXaGljaCBzdGFydHMgd2l0aCBubyBwcmVkaWN0b3JzIGluIHRoZSBtb2RlbCwgSXRlcmF0aXZlbHkgYWRkcyB0aGUgbW9zdCBjb250cmlidXRpdmUgcHJlZGljdG9yLCBhbmQgc3RvcHMgd2hlbiB0aGUgaW1wcm92ZW1lbnQgaXMgbm8gbG9uZ2VyIHN0YXRpc3RpY2FsbHkgc2lnbmlmaWNhbnQuCgoyLiAqKkJhY2t3b2FyZCBzZWxlY3Rpb24ob3IgYmFja3dhcmQgZWxpbWluYXRpb24pKio6IHdoaWNoIHN0YXJ0cyB3aXRoIGFsbCBwcmVkaWN0b3JzIGluIHRoZSBtb2RlbCAoZnVsbCBtb2RlbCksIGl0ZXJhdGl2ZWx5IHJlbW92ZXMgdGhlIGxlYXN0IGNvbnRyaWJ1dGl2ZSBwcmVkaWN0b3JzLCBhbmQgc3RvcHMgd2hlbiB5b3UgaGF2ZSBhIG1vZGVsIHdoZXJlIGFsbCBwcmVkaWN0b3JzIGFyZSBzdGF0aXN0aWNhbGx5IHNpZ25pZmljYW50LgoKMy4gKipTdGVwd2lzZSBzZWxlY3Rpb24gKG9yIHNlcXVlbnRpYWwgcmVwbGFjZW1lbnQpKiogd2hpY2ggaXMgYSBjb21iaW5hdGlvbiBvZiBmb3J3YXJkIGFuZCBiYWNrd2FyZCBzZWxlY3Rpb25zLiB5b3Ugc3RhcnQgd2l0aCBubyBwcmVkaWN0b3JzLCB0aGVuIHNlcXVlbnRpYWxseSBhZGQgdGhlIG1vc3QgY29udHJpYnV0aXZlIHByZWRpY3RvcnMgKGxpa2UgZm9yd2FyZCBzZWxlY3Rpb24pLiBBZnRlciBhZGRpbmcgZWFjaCBuZXcgdmFyaWFibGUsIHJlbW92ZSBhbnkgdmFyaWFibGVzIHRoYXQgbm8gbG9uZ2VyIHByb3ZpZGUgYW5kIGltcHJvdmVtZW50IGluIHRoZSBtb2RlbCBmaXQKCmBgYHtyfQpsaWJyYXJ5KHRpZHl2ZXJzZSkKbGlicmFyeShjYXJldCkKbGlicmFyeShsZWFwcykKYGBgCgoKIyMjIENvbXB1dGluZyBzZXRwd2lzZSByZWdyZXNzaW9uCgpgYGB7cn0KbGlicmFyeShNQVNTKQojIEZpdCB0aGUgZnVsbCBtb2RlbApmdWxsLm1vZGVsIDwtIGxtKEZlcnRpbGl0eSB+LiwgZGF0YSA9IHN3aXNzKQoKIyBTdGVwd2lzZSByZWdyZXNzaW9uIG1vZGVsCnN0ZXAubW9kZWwgPC0gc3RlcEFJQyhmdWxsLm1vZGVsLCBkaXJlY3Rpb24gPSAiYm90aCIsIHRyYWNlID0gRkFMU0UpCgpzdW1tYXJ5KHN0ZXAubW9kZWwpCmBgYAoKCiMjIFBlbmFsaXplZCBSZWdyZXNzaW9uOiBSaWRnZSwgTGFzc28gYW5kIEVsYXN0aWMgTmV0CgoKIyMjIFNocmlua2FnZSBtZXRob2RzCgojIyMgUmlkZ2UgcmVncmVzc2lvbgoKUmlkZ2UgcmVncmVzc2lvbiBzaHJpbmtzIHRoZSByZWdyZXNzaW9uIGNvZWZmaWNpZW50cywgc28gdGhhdCB2YXJpYmxlcywgd2l0aCBtaW5vciBjb250cmlidXRpb24gdG8gdGhlIG91dGNvbWUsIGhhdmUgdGhlaXIgY29lZmZpY2llbnRzIGNsb3NlIHRvIHplcm8uCgpUaGUgc2hyaW5rYWdlIG9mIHRoZSBjb2VmZmljaWVudHMgaXMgYWNoaWV2ZWQgYnkgcGVuYWxpemluZyB0aGUgcmVncmVzc2lvbiBtb2RlbCB3aXRoIGEgcGVuYWx0eSB0ZXJtIGNhbGxlZCAqKkwyLW5vcm0qKiwgd2hpY2ggaXMgdGhlIHN1bSBvZiB0aGUgc3F1YXJlZCBjb2VmZmljaWVudHMuCgpUaGUgYW1vdW50IG9mIHRoZSBwZW5hbHR5IGNhbiBiZSBmaW5lLXR1bmVkIHVzaW5nIGEgY29uc3RhbnQgY2FsbGVkIGxhbWJkYSAuIFNlbGVjdGluZyBhIGdvb2QgdmFsdWUgZm9yIGxhbWJkYSBpcyBjcml0aWNhbAoKV2hlbiBsYW1iZGEgPSAwLCB0aGUgcGVuYWx0eSB0ZXJtIGhhcyBubyBlZmZlY3QsIGFuZCByaWRnZSByZWdyZXNzaW9uIHdpbGwgcHJvZHVjZSB0aGUgY2Fsc3NpY2FsIGxlYXN0IHNxdWFyZSBjb2VmZmljaWVudHMuIEhvd2V2ZXIsIGFzIGxhbWJkYSBpbmNyZWFzZXMgdG8gaW5maW5pdGUsIHRoZSBpbXBhY3Qgb2YgdGhlIHNocmlua2FnZSBwZW5hbHR5IGdyb3dzLCBhbmQgdGhlIHJpZGdlIHJlZ3Jlc3Npb24gY29lZmZpY2llbnRzIHdpbGwgZ2V0IGNsb3NlIHplcm8uCgo+IE5vdGUgdGhhdCwgaW4gY29udHJhc3QgdG8gdGhlIG9yZGluYXJ5IGxlYXN0IHNxdWFyZSByZWdyZXNzaW9uLCByaWRnZSByZWdyZXNzaW9uIGlzIGhpZ2hseSBhZmZlY3RlZCBieSB0aGUgc2NhbGUgb2YgdGhlIHByZWRpY3RvcnMuIFRoZXJmb3JlLCBpdCBpcyBiZXR0ZXIgdG8gc3RhbmRhcmRpemUgKGkuZS4sIHNjYWxlKSB0aGUgcHJlZGljdG9ycyBiZWZvcmUgYXBwbHlpbmcgdGhlIHJpZGdlIHJlZ3Jlc3Npb24sIHNvIHRoYXQgYWxsIHRoZSBwcmVkaWN0b3JzIGFyZSBvbiB0aGUgc2FtZSBzY2FsZS4KClRoZSBzdGFuZGFyZGl6YXRpb24gb2YgYSBwcmVkaWN0b3IgeCwgY2FuIGJlIGFjaGlldmVkIHVzaW5nIHRoZSBmb3JtdWxhIHhgID0geC9zZCh4KSwgd2hlcmUgc2QoeCkgaXMgdGhlIHN0YWRhcmQgZGV2aWF0aW9uIG9mIC4gVGhlIGNvbnNlcXVlbmNlIG9mIHRoaXMgaXMgdGhhdCwgYWxsIHN0YW5kYXJkaXplZCBwcmVkaWN0b3JzIHdpbGwgaGF2ZSBhIHN0YW5kYXJkIGRldmlhdGlvbiBvZiBvbmUgYWxsb3dpbmcgdGhlIGZpbmFsIGZpdCB0byBub3QgZGVwZW5kIG9uIHRoZSBzY2FsZSBvbiB3aGljaCB0aGUgcHJlZGljdG9ycyBhcmUgbWVhc3VyZWQuCgpPbmUgaW1wb3J0YW50IGFkdmFudGFnZSBvZiB0aGUgcmlkZ2UgcmVncmVzc2lvbiwgaXMgdGhhdCBpdCBzdGlsbCBwZXJmcm9tcyB3aWxsLCBjb21wYXJlZCB0byB0aGUgb3JkaW5hcnkgbGVhc3Qgc3F1YXJlIG1ldGhvZCwgaW4gYSBzaXR1YXRpb24gd2hlcmUgeW91IGhhdmUgYSBsYXJnZSBtdWx0aXZhcmlhdGUgZGF0YSB3aXRoIHRoZSBudW1iZXIgb2YgcHJlZGljdG9ycyAocCkgbGFyZ2VyIHRoYW4gdGhlIG51bWJlciBvZiBvYnNlcnZhdGlvbnMgKG4pLgoKT25lIGRpc2FkdmFudGFnZSBvZiB0aGUgcmlkZ2UgcmVncmVzc2lvbiBpcyB0aGF0LCBpdCB3aWxsIGluY2x1ZGVkIGFsbCB0aGUgcHJlZGljdG9ycyBpbiB0aGUgZmluYWwgbW9kZWwsIHVubGlrZSB0aGUgc3RlcHdpc2UgcmVncmVzc2lvbiBtZXRob2RzLCB3aGljaCB3aWxsIGdlbnJhbGx5IHNlbGVjdCBtb2RlbHMgdGhhdCBpbnZvbHZlIGEgcmVkdWNlZCBzZXQgb2YgdmFyaWFibGVzLgoKUmlkZ2UgcmVncmVzc2lvbiBzaHJpbmtzIHRoZSBjb2VmZmljaWVudHMgdG93YXJkcyB6ZXJvLCBidXQgaXQgd2lsbCBub3Qgc2V0IGFueSBvZiB0aGVtIGV4YWN0bHkgdG8gemVyby4gVGhlIGxhc3NvIHJlZ3Jlc3Npb24gaXMgYW4gYWx0ZXJuYXRpdmUgdGhhdCBvdmVyY29tZXMgdGhpcyBkcmF3YmFjay4KCgojIyMgTGFzc28gcmVncmVzc2lvbgoKTGFzc28gc3RhbmRzIGZvciAqKkxlYXN0IEFic29sdXRlIFNocmlua2FnZSBhbmQgU2VsZWN0aW9uIE9wZXJhdG9yKiouIEl0IHNocmlua3MgdGhlIHJlZ3Jlc3Npb24gY29lZmZpY2llbnRzIHRvd2FyZCB6ZXJvIGJ5IHBlbmFsaXppbmcgdGhlIHJlZ3Jlc3Npb24gbW9kZWwgd2l0aCBhIHBlbmFsdHkgdGVybSBjYWxsZWQgKipMMS1ub3JtKiosIHdoaWNoIGlzIHRoZSBzdW0gb2YgdGhlIGFic29sdXRlIGNvZWZmaWNpZW50cy4KCkluIHRoZSBjYXNlIG9mIGxhc3NvIHJlZ3Jlc3Npb24sIHRoZSBwZW5hbHR5IGhhcyB0aGUgZWZmZWN0IG9mIGZvcmNpbmcgc29tZSBvZiB0aGUgY29lZmZpY2llbnQgZXN0aW1hdGVzLCB3aXRoIGEgbWlub3IgY29udHJpYnV0aW9uIHRvIHRoZSBtb2RlbCwgdG8gYmUgZWNhY3RseSBlcXVhbCB0byB6ZXJvLiBUaGlzIG1lYW5zIHRoYXQsIGxhc3NvIGNhbiBiZSBhbHNvIHNlZW4gYXMgYW4gYWx0ZXJuYXRpdmUgdG8gdGhlIHN1YnNldCBzZWxlY3Rpb24gbWV0aG9kcyBmb3IgcGVyZm9ybWluZyB2YXJpYWJsZSBzZWxlY3Rpb24gaW4gb3JkZXIgdG8gcmVkdWNlIHRoZSBjb21wbGV4aXR5IG9mIHRoZSBtb2RlbC4KCkFzIGluIHJpZGdlIHJlZ3Jlc3Npb24sIHNlbGVjdGluZyBhIGdvb2QgdmFsdWUgb2YgbGFtYmRhIGZvciB0aGUgbGFzc28gaXMgY3JpdGljYWwuCgpPbmUgb2J2aW91cyBhZHZhbnRhZ2Ugb2YgbGFzc28gcmVncmVzc2lvbiBvdmVyIHJpZGdlIHJlZ3Jlc3Npb24sIGlzIHRoYXQgaXQgcHJvZHVjZXMgc2ltcGxlciBhbmQgbW9yZSBpbnRlcnByZXRhYmxlIG1vZGVscyB0aGF0IGluY29ycG9yYXRlIG9ubHkgYSByZWR1Y2VkIHNldCBvZiB0aGUgcHJlZGljdG9ycy4gSG93ZXZlciwgbmVpdGhlciByaWRnZSByZWdyZXNzaW9uIG5vciB0aGUgbGFzc28gd2lsbCB1bml2ZXJzYWxseSBkb21pbmF0ZSB0aGUgb3J0aGVyLgoKR2VuZXJhbGx5LCBsYXNzbyBtZ2h0IHBlcmZvcm0gYmV0dGVyIGluIGEgc2l0dWF0aW9uIHdoZXJlIHNvbWUgb2YgdGhlIHByZWRpY3RvcnMgaGF2ZSBsYXJnZSBjb2VmZmljaWVudHMsIGFuZCB0aGUgcmVtYWluaW5nIHByZWRpY3RvcnMgaGF2ZSB2ZXJ5IHNtYWxsIGNvZWZmaWNpZW50cy4KClJpZGdlIHJlZ3Jlc3Npb24gd2lsbCBwZXJmb3JtIGJldHRlciB3aGVuIHRoZSBvdXRjb21lIGlzIGEgZnVuY3Rpb24gb2YgbWFueSBwcmVkaWN0b3JzLCBhbGwgd2l0aCBjb2VmZmljZW50cyBvZiByb3VnaGx5IGVxdWFsIHNpemUuCgpDcm9zcy12YWxpZGF0aW9uIG1ldGhvZHMgY2FuIGJlIHVzZWQgZm9yIGlkZW50aWZ5aW5nIHdoaWNoIG9mIHRoZXNlIHR3byB0ZWNobmlxdWVzIGlzIGJldHRlciBvbiBhIHBhcnRpY3VsYXIgZGF0YSBzZXQuCgoKCiMjIyBFbGFzdGljIE5ldAoKRWxhc3RpYyBOZXQgcHJvZHVjZXMgYSByZWdyZXNzaW9uIG1vZGVsIHRoYXQgaXMgcGVuYWxpemVkIHdpdGggYm90aCB0aGUgKipMMS1ub3JtKiogYW5kICoqTDItbm9ybSoqLiBUaGUgY29uc2VxdWVuY2Ugb2YgdGhpcyBpcyB0byBlZmZlY3RpdmVseSBzaHJpbmsgY29lZmZpY2VudHMgKExpa2UgaW4gcmlkZ2UgcmVncmVzc2lvbikgYW5kIHRvIHNldCBzb21lIGNvZWZmaWNpZW50cyB0byB6ZXJvIChhcyBpbiBMQVNTTykuCgojIyMjIExvYWRpbmcgcmVxdWlyZWQgUiBwYWNrYWdlcwoKCmBgYHtyfQpsaWJyYXJ5KHRpZHl2ZXJzZSkKbGlicmFyeShjYXJldCkKbGlicmFyeShnbG1uZXQpCgpgYGAKCgpQcmVwYXJpbmcgdGhlIGRhdGEKCldlJ2xsIHVzZSB0aGUgQm9zdG9uIGRhdGEgc2V0IFtpbiBNQVNTIHBhY2thZ2VdLCBmb3IgcHJlZGljdGluZyB0aGUgbWVkaWFuIGhvdXNlIHZhbHVlKG1kZXYpLCBpbiBCb3N0b24gU3VidXJicywgYmFzZWQgb24gbXVsdGlwbGUgcHJlZGljdG9yIHZhcmlhYmxlcy4KCgpgYGB7cn0KIyBMb2FkIHRoZSBkYXRhCmRhdGEoIkJvc3RvbiIsIHBhY2thZ2UgPSAiTUFTUyIpCgojIFNwbGl0IHRoZSBkYXRhIGludG8gdHJhaW5pbmcgYW5kIHRlc3Qgc2V0CnNldC5zZWVkKDEyMykKYm9zdG9uLnNhbXBsZXMgPC0gQm9zdG9uJG1lZHYgJT4lCiAgY3JlYXRlRGF0YVBhcnRpdGlvbihwPTAuOCwgbGlzdCA9IEZBTFNFKQoKdHJhaW4uYm9zdG9uLmRhdGEgPC0gQm9zdG9uW2Jvc3Rvbi5zYW1wbGVzLCBdCnRlc3QuYm9zdG9uLmRhdGEgPC0gQm9zdG9uWy1ib3N0b24uc2FtcGxlcywgXQpgYGAKCgojIyMjIENvbXB1dGluZyBwZW5hbGl6ZWQgbGluZWFyIHJlZ3Jlc3Npb24KCllvdSBuZWVkIHRvIGNyZWF0ZSB0d28gb2JqZWN0czoKCiogeSBmb3Igc3RvcmluZyB0aGUgb3V0Y29tZSB2YXJpYWJsZQoqIHggZm9yIGhvbGRpbmcgdGhlIHByZWRpY3RvciB2YXJpYWJsZXMuCgpgYGB7cn0KeCA8LSBtb2RlbC5tYXRyaXgobWVkdn4uLCB0cmFpbi5ib3N0b24uZGF0YSlbLC0xXQojIE91dGNvbWUgdmFyaWFibGUKeSA8LSB0cmFpbi5ib3N0b24uZGF0YSRtZWR2CmBgYAoKV2UnbGwgdXNlIHRoZSBSIGZ1bmN0aW9uIGdsbW5ldCgpIGZybyBjb21wdXRpbmcgcGVuYWxpemVkIGxpbmVhciByZWdyZXNzaW9uIG1vZGVscy4KCgpgYGB7cn0KZ2xtbmV0KHgseSwgYWxwaGEgPSAxLCBsYW1iZGEgPSBOVUxMKQpgYGAKCiogeDogbWF0cml4IG9mIHByZWRpY3RvciB2YXJpYWJsZXMKKiB5OiB0aGUgcmVzcG9uc2Ugb3Igb3V0Y29tZSB2YXJpYWJsZSwgd2hpY2ggaXMgYSBiaW5hcnkgdmFyaWFibGUuCiogYWxwaGE6IHRoZSBlbGFzdGljbmV0IG1peGluZyBwYXJhbWV0ZXIuIEFsbG93ZWQgdmFsdWVzIGluY2x1ZGU6CiAgIC0gIjEiOiBmb3IgbGFzc28gcmVncmVzc2lvbgogICAtICIwIjogZm9yIHJpZGdlIHJlZ3Jlc3Npb24KICAgLSBhIHZhbHVlZCBiZXR3ZWVuIDAgYW5kIDEgKHNheSAwLjI1KSBmb3IgZWxhc3RpYyBuZXQgcmVncmVzc2lvbi4KICAgLSBsYW1iZGE6IGEgbnVtZXJpYyB2YWx1ZSBkZWZpbmluZyB0aGUgYW1vdW50IG9mIHNocmlua2FnZS4gU2hvdWxkIGJlIHNwZWNpZnkgYnkgYW5hbHlzdAoKCkluIHBlbmFsaXplZCByZWdyZXNzaW9uLCB5b3UgbmVlZCB0byBzcGVjaWZ5IGEgY29uc3RhbnQgbGFtYmRhIHRvIGFkanVzdCB0aGUgYW1vdW50IG9mIHRoZSBjb2VmZmljZW50IHNocmlua2FnZS4gVGhlIGJlc3QgbGFtYmRhIGZvciB5b3VyIGRhdGEsIGNhbiBiZSBkZWZpbmVkIGFzIHRoZSBsYW1iZGEgdGhhdCBtaW5pbWl6ZSB0aGUgY3Jvc3MtdmFsaWRhdGlvbiBwcmVkaWN0aW9uIGVycm9yIHJhdGUuIFRoaXMgY2FuIGJlIGRldGVybWluZWQgYXV0b21hdGljYWxseSB1c2luZyB0aGUgZnVuY3Rpb24gY3YuZ2xtbmV0KCkuCgo+SW4gdGhlIGZvbGxvd2luZyBzZWN0aW9uLCB3ZSBzdGFydCBieSBjb21wdXRpbmcgcmlkZ2UsIGxhc3NvIGFuZCBlbGFzdGljIG5ldCByZWdyZXNpb24gbW9kZWxzLiBOZXh0LCB3ZSdsbCBjb21wYXJlIHRoZSBkaWZmZXJlbnQgbW9kZWxzIGluIG9yZGVyIHRvIGNob29zZSB0aGUgYmVzdCBvbmUgZm9yIG91ciBkYXRhClRoZSBiZXN0IG1vZGVsIGlzIGRlZmluZWQgYXMgdGhlIG1vZGVsIHRoYXQgaGFzIHRoZSBsb3dlc3QgcHJlZGljdGlvbiBlcnJvciwgUk1TRQoKQ29tcHV0aW5nIHJpZGdlIHJlZ3Jlc3Npb24KCmBgYHtyfQojIEZpbmQgdGhlIGJlc3QgbGFtYmRhIHVzaW5nIGNyb3NzLXZhbGlkYXRpb24Kc2V0LnNlZWQoMTIzKQpjdiA8LSBjdi5nbG1uZXQoeCx5LCBhbHBoYSA9IDApCgojIERpc3BsYXkgdGhlIGJlc3QgbGFtYmRhIHZhbHVlCmN2JGxhbWJkYS5taW4KYGBgCgoKYGBge3J9CiMgRml0IHRoZSBmaW5hbCBtb2RlbCBvbiB0aGUgdHJhaW5pbmcgZGF0YQptb2RlbC5yaWRnZSA8LSBnbG1uZXQoeCx5LCBhbHBoYSA9IDAsIGxhbWJkYSA9IGN2JGxhbWJkYS5taW4pCgojIERpc3BsYXkgcmVncmVzc2lvbiBjb2VmZmljaWVudHMKY29lZihtb2RlbC5yaWRnZSkKCmBgYAoKCgpgYGB7cn0KIyBNYWtlIHByZWRpY3Rpb25zIG9uIHRoZSB0ZXN0IGRhdGEKeC50ZXN0IDwtIG1vZGVsLm1hdHJpeChtZWR2fi4sIHRlc3QuYm9zdG9uLmRhdGEpWywtMV0KcHJlZGljdGlvbnMucmlkZ2UgPC0gbW9kZWwucmlkZ2UgJT4lIHByZWRpY3QoeC50ZXN0KSAlPiUgYXMudmVjdG9yKCkKCiMgTW9kZWwgcGVyZm9ybWFuY2UgbWV0cmljcwoKZGF0YS5mcmFtZSgKICBSTVNFID0gUk1TRShwcmVkaWN0aW9ucy5yaWRnZSwgdGVzdC5ib3N0b24uZGF0YSRtZWR2KSwKICBSc3F1YXJlID0gUjIocHJlZGljdGlvbnMucmlkZ2UsIHRlc3QuYm9zdG9uLmRhdGEkbWVkdikKKQpgYGAKCj4gTm90ZSB0aGF0IGJ5IGRlZmF1bHQsIHRoZSBmdW5jdGlvbiBnbG1uZXQoKSBzdGFuZGFyZGl6ZXMgdmFyaWFibGVzIHNvIHRoYXQgdGhlaXIgc2NhbGVzIGFyZSBjb21wYXJhYmxlLiBIb3dldmVyLCB0aGUgY29lZmZpY2llbnRzIGFyZSBhbHdheXMgcmV0dXJuZWQgb24gdGhlIG9yaWdpbmFsIHNjYWxlLgoKQ29tcHV0aW5nIGxhc3NvIHJlZ3Jlc3Npb24KClRoZSBvbmx5IGRpZmZlcmVuY2UgYmV0d2VlbiB0aGUgUiBjb2RlIHVzZWQgZm9yIHJpZGdlIHJlZ3Jlc3Npb24gaXMgdGhhdCBmb3IgbGFzc28gcmVncmVzc2lvbiB5b3UgbmVlZCB0byBzcGVjaWZ5IHRoZSBhcmd1bWVudCBhbHBoYSA9IDEgaW5zdGVhZCBvZiBhbHBoYSAtIDAgCgoKYGBge3J9CiMgRmluZCB0aGUgYmVzdCBsYW1iZGEgdXNpbmcgY3Jvc3MtdmFsaWRhdGlvbgpzZXQuc2VlZCgxMjMpCmN2IDwtIGN2LmdsbW5ldCh4LHksYWxwaGEgPSAxKQojIERpc3BsYXkgdGhlIGJlc3QgbGFtYmRhIHZhbHVlCmN2JGxhbWJkYS5taW4KYGBgCgpgYGB7cn0KIyBGaXQgdGhlIGZpbmFsIG1vZGVsIG9uIHRoZSB0cmFpbmluZyBkYXRhCm1vZGVsIDwtIGdsbW5ldCh4LHksIGFscGhhID0gMSwgbGFtYmRhID0gY3YkbGFtYmRhLm1pbikKI0Rpc3BsYXkgcmVncmVzc2lvbiBjb2VmZmljaWVudHMKY29lZihtb2RlbCkKYGBgCgoKYGBge3J9CiMgTWFrZSBwcmVkaWN0aW9ucyBvbiB0aGUgdGVzdCBkYXRhCngubGFzby50ZXN0IDwtIG1vZGVsLm1hdHJpeChtZWR2IH4uLCB0ZXN0LmJvc3Rvbi5kYXRhKVssLTFdCnByZWRpY3Rpb25zLmxhc3NvIDwtIG1vZGVsICU+JSBwcmVkaWN0KHgudGVzdCkgJT4lIGFzLnZlY3RvcigpCgojIE1vZGVsIHBlcmZvcm1hbmNlIG1ldHJpY3MKZGF0YS5mcmFtZSgKICBSTVNFID0gUk1TRShwcmVkaWN0aW9ucy5sYXNzbywgdGVzdC5ib3N0b24uZGF0YSRtZWR2KSwKICBSc3F1YXJlID0gUjIocHJlZGljdGlvbnMubGFzc28sIHRlc3QuYm9zdG9uLmRhdGEkbWVkdikKKQpgYGAKCiMjIyBDb21wdXRpbmcgZWxhc3RpYyBuZXQgcmVncmVzc2lvbgoKVGhlIGVsYXN0aWMgbmV0IHJlZ3Jlc3Npb24gY2FuIGJlIGVhc2lseSBjb21wdXRlZCB1c2luZyB0aGUgY2FyZXQgd29ya2Zsb3csIHdoaWNoIGludm9rZXMgdGhlIGdsbW5ldCBwYWNrYWdlLgoKV2UgdXNlIGNhcmV0IHRvIGF1dG9tYXRpY2FsbHkgc2VsZWN0IHRoZSBiZXN0IHR1bmluZyBwYXJhbWV0ZXJzIGFscGhhIGFuZCBsYW1iZGEuIHRoZSBjYXJldCBwYWNrYWdlcyB0ZXN0cyBhIHJhbmdlIG9mIHBvc3NpYmxlIGFscGhhIGFuZCBsYW1iZGEgdmFsdWVzLCB0aGVuIHNlbGVjdHMgdGhlIGJlc3QgdmFsdWVzIGZybyBsYW1iZGEgYW5kIGFscGhhLCByZXN1bHRpbmcgdG8gYSBmaW5hbCBtb2RsIHRoYXQgaXMgYW4gZWxhc3RpYyBuZXQgbW9kbGUuCgpIZXJlLCB3ZSdsbCB0ZXN0IHRoZSBjb21iaW5hdGlvbiBvZiAxMCBkaWZmZXJlbnQgdmFsdWVzIGZvciBhbHBoYSBhbmQgbGFtYmRhLiBUaGlzIGlzIHNwZWNpZmllZCB1c2luZyB0aGUgb3B0aW9uIHR1bmVMZW5ndGguCgpUaGUgYmVzdCBhbHBoYSBhbmQgbGFtYmRhIHZhbHVlcyBhcmUgdGhvc2UgdmFsdWVzIHRoYXQgbWluaW1pemUgdGhlIGNyb3NzLXZhbGlkYXRpb24gZXJyb3IKCgpgYGB7cn0KIyAjIEJ1aWxkIHRoZSBtb2RlbCB1c2luZyB0aGUgdHJhaW5pbmcgc2V0CiMgc2V0LnNlZWQoMTIzKQojIG1vZGVsLmVsYXN0aWMgPC0gdHJhaW4oCiMgICBtZWR2IH4uLCBkYXRhID0gdHJhaW4uYm9zdG9uLmRhdGEsIG1ldGhvZCA9ICJnbG1uZXQiLAojICAgdHJjb250cm9sID0gdHJhaW5Db250cm9sKCJjdiIsIG51bWJlciA9IDUpLAojICAgdHVuZUxlbmd0aCA9IDEwCiMgKQojIAojICMgQmVzdCB0dW5pbmcgcGFyYW1ldGVyCiMgbW9kZWwuZWxhc3RpYyRiZXN0VHVuZQpgYGAKCgojIyBQcmluY2lwYWwgQ29tcG9uZW50IGFuZCBQYXJ0aWFsIExlYXN0IFNxdWFyZXMgUmVncmVzc2lvbgoKCiMjIyBQcmluY2lwYWwgY29tcG9uZW50IHJlZ3Jlc3Npb24KClRoZSBwcmluY2lwYWwgY29tcG9uZW50IHJlZ3Jlc3Npb24oUENSKSBmaXJzdCBhcHBsaWVzIFByaW5jaXBhbCBDb21wb25lbnQgQW5hbHlzaXMgb24gdGhlIGRhdGEgc2V0IHRvIHN1bW1hcml6ZSB0aGUgb3JpZ2luYWwgcHJlZGljdG9yIHZhcmlhYmxlcyBpbnRvIGZldyBuZXcgdmFyaWFibGVzIGFsc28ga25vd24gYXMgcHJpbmNpcGFsIGNvbXBvbmVudCAoUENzKSwgd2hpY2ggYXJlIGEgbGluZWFyIGNvbWJpbmF0aW9uIG9mIHRoZSBvcmlnaW5hbCBkYXRhLgoKVGhlc2UgUENzIGFyZSB0aGVuIHVzZWQgdG8gYnVpbGQgdGhlIGxpbmVhciByZWdyZXNzaW9uIG1vZGVsLiBUaGUgbnVtYmVyIG9mIHByaW5jaXBhbCBjb21wb25lbnRzLCB0byBpbmNvcnBvcmF0ZSBpbiB0aGUgbW9kZWwsIGlzIGNob3NlbiBieSBjb3Jzcy12YWxpZGF0aW9uIChjdikuIE5vdGUgdGhhdCwgUENSIGlzIHN1aXRhYmxlIHdoZW4gdGhlIGRhdGEgc2V0IGNvbnRhaW5zIGhpZ2hseSBjb3JyZWxhdGVkIHByZWRpY3RvcnMuCgojIyMgUGFydGlhbCBsZWFzdCBzcXVhcmVzIHJlZ3Jlc3Npb24KCkEgcG9zc2libGUgZHJhd2JhY2sgb2YgUENSIGlzIHRoYXQgd2UgaGF2ZSBubyBndWFyYW50ZWUgdGhhdCB0aGUgc2VsZWN0ZWQgcHJpbmNpcGFsIGNvbXBvbmVudHMgYXJlIGFzc29jaWF0ZWQgd2l0aCB0aGUgb3V0Y29tZS4gSGVyZSwgdGhlIHNlbGVjdGlvbiBvZiB0aGUgcHJpbmNpcGFsIGNvbXBvbmVudHMgdG8gaW5jb3Jwb3JhdGUgaW4gdGhlIG1vZGVsIGlzIG5vdCBzdXBlcnZpc2VkIGJ5IHRoZSBvdXRjb21lIHZhcmlhYmxlLgoKQW4gYWx0ZXJuYXRpdmUgdG8gUENSIGlzIHRoZSAqKlBhcnRpYWwgTGVhc3QgU3F1YXJlcyoqIChQTFMpIHJlZ3Jlc3Npb24sIHdoaWNoIGlkZW50aWZpZXMgbmV3IHByaW5jaXBhbCBjb21wb25lbnRzIHRoYXQgbm90IG9ubHkgc3VtbWFyaXplcyB0aGUgb3JpZ2luYWwgcHJlZGljdG9ycywgYnV0IGFsc28gdGhhdCBhcmUgcmVsYXRlZCB0byB0aGUgb3V0Y29tZS4gVGhlc2UgY29tcG9uZW50cyBhcmUgdGhlbiB1c2VkIHRvIGZpdCB0aGUgcmVncmVzc2lvbiBtb2RlbC4gU28gY29tcGFyZWQgUENSLCBQTFMgdXNlcyBhIGRpbWVuc2lvbiByZWR1Y3Rpb24gc3RyYXRlZ3kgdGhhdCBpcyBzdXBlcnZpc2VkIGJ5IHRoZSBvdXRjb21lLgoKTGlrZSBQQ1IsIFBMUyBpcyBjb252ZW5pZW50IGZvciBkYXRhIHdpdGggaGlnaGx5LWNvcnJlbGF0ZWQgcHJlZGljdG9ycy4gVGhlIG51bWJlciBvZiBQQ3MgdXNlZCBpbiBQTFMgaXMgZ2VuZXJhbGx5IGNob3NlbiBieSBjcm9zcy12YWxpZGF0aW9uLiBQcmVkaWN0b3JzIGFuZCB0aGUgb3V0Y29tZSB2YXJpYWJsZXMgc2hvdWxkIGJlIGdlbmVyYWxseSBzdGFuZGFyZGl6ZWQsIHRvIG1ha2UgdGhlIHZhcmlhYmxlcyBjb21wYXJhYmxlLgoKCiMjIyBMb2FkaW5nIHJlcXVpcmVkIFIgcGFja2FnZXMKCiogdGlkeXZlcnNlIGZvciBlYXN5IGRhdGEgbWFuaXB1bGF0aW9uIGFuZCB2aXN1YWxpemF0aW9uCiogY2FyZXQgZm9yIGVhc3kgbWFjaGluZSBsZWFybmluZyB3b3JrZmxvdwoqIHBscywgZm9yIGNvbXB1dGluZyBQQ1IgYW5kIFBMUwoKYGBge3J9CmxpYnJhcnkodGlkeXZlcnNlKQpsaWJyYXJ5KGNhcmV0KQpsaWJyYXJ5KHBscykKYGBgCgoKIyMjIFByZXBhcmluZyB0aGUgZGF0YQoKV2UnbGwgdXNlIHRoZSBib3N0b24gZGF0YSBzZXQKCiMjIyMgQ29tcHV0aW5nIHByaW5jaXBhbCBjb21wb25lbnQgcmVncmVzc2lvbgoKCmBgYHtyfQojIEJ1aWxkIHRoZSBtb2RlbCBvbiB0cmFpbmluZyBzZXQKCnNldC5zZWVkKDEyMykKCm1vZGVsLnBjIDwtIHRyYWluKAogIG1lZHZ+LiwgZGF0YSA9IHRyYWluLmJvc3Rvbi5kYXRhLCBtZXRob2QgPSAicGNyIiwKICBzY2FsZSA9IFRSVUUsCiAgdHJDb250cm9sID0gdHJhaW5Db250cm9sKCJjdiIsIG51bWJlciA9IDEwKSwKICB0dW5lTGVuZ3RoID0gMTAKKQoKIyBQbG90IG1vZGVsIFJNU0UgdnMgZGlmZmVyZW50IHZhbHVlcyBvZiBjb21wb25lbnRzCgpwbG90KG1vZGVsLnBjKQoKI1ByaW50IHRoZSBiZXN0IHR1bm5pbmcgcGFyYW1ldGVyIG5jb21wIHRoYXQgCiMgbWluaW1pemUgdGhlIGNyb3NzLXZhbGlkYXRpb24gZXJyb3IsIFJNU0UKbW9kZWwucGMkYmVzdFR1bmUKYGBgCgoKCmBgYHtyfQpzdW1tYXJ5KG1vZGVsLnBjJGZpbmFsTW9kZWwpCmBgYAoKYGBge3J9CiMgTWFrZSBwcmVkaWN0aW9ucwpwcmVkaWN0aW9ucy5wYyA8LSBtb2RlbC5wYyAlPiUgcHJlZGljdCh0ZXN0LmJvc3Rvbi5kYXRhKQojIE1vZGVsIHBlcmZvcm1hbmNlIG1ldHJpY3MKCmRhdGEuZnJhbWUoCiAgUk1TRSA9IGNhcmV0OjpSTVNFKHByZWRpY3Rpb25zLnBjLCB0ZXN0LmJvc3Rvbi5kYXRhJG1lZHYpLAogIFJzcXVhcmUgPSBjYXJldDo6UjIocHJlZGljdGlvbnMucGMsIHRlc3QuYm9zdG9uLmRhdGEkbWVkdikKKQpgYGAKClRoZSBwbG90IHNob3dzIHRoZSBwcmVkaWN0aW9uIGVycm9yIG1hZGUgYnkgdGhlIG1vZGVsIGFjY2NvcmRpbmcgdG8gdGhlIG51bWJlciBvZiBwcmluY2lwYWwgY29tcG9uZW50cyBpbmNvcnBvcmF0ZWQgaW4gdGhlIG1vZGVsLgoKT3VyIGFuYWx5c2lzIHNob3dzIHRoYXQsIGNob29zaW5nIGZpdmUgcHJpbmNpcGFsIGNvbXBvbmVudHMgKG5jb21wID0gNSkgZ2l2ZXMgdGhlIHNtYWxsZXN0IHByZWRpY3Rpb24gZXJyb3IgUk1TRS4KClRoZSAqKnN1bW1hcnkoKSoqIGZ1bmN0aW9uIGFsc28gcHJvdmlkZXMgdGhlIHBlcmNlbnRhZ2Ugb2YgdmFyaWFuY2UgZXhwbGFpbmVkIGluIHRoZSBwcmVkaWN0b3JzKHgpIGFuZCBpbiB0aGUgb3V0Y29tZSAobWVkdikgdXNpbmcgZGlmZmVyZW50IG51bWJlcnMgb2YgY29tcG9uZW50cy4KCkZvciBleGFtcGxlLCA4MC45NCUgb2YgdGhlIHZhcmlhdGlvbiAob3IgaW5mb3JtYXRpb24pIGNvbnRhaW5lZCBpbiB0aGUgcHJlZGljdG9ycyBhcmUgY2FwdHVyZWQgYnkgNSBjb21wb25lbnRzIChuY29tcCA9IDUpLiBBZGRpdGlvbmFsbHksIHNldHRpbmcgbmNvbXAgPSA1LCBjYXB0dXJlcyA3MSUgb2YgdGhlIGluZm9ybWF0aW9uIGluIHRoZSBvdXRjb21lIHZhcmlhYmxlIChtZWR2KSwgd2hpY2ggaXMgZ29vZAoKPiBUYWtlbiB0b2dldGhlciwgY3Jvc3MtdmFsaWRhdGlvbiBpZGVudGlmaWVzIG5jb21wIC0gNSBhcyB0aGUgb3B0aW1hbCBudW1iZXIgb2YgUENzIHRoYXQgbWluaW1pemUgdGhlIHByZWRpY3Rpb24gZXJyb3IgKFJNU0UpIGFuZCBleHBsYWlucyBlbm91Z2ggdmFyaWF0aW9uIGluIHRoZSBwcmVkaWN0b3JzIGFuZCBpbiB0aGUgb3V0Y29tZQoKCgojIyMgQ29tcHV0aW5nIHBhcnRpYWwgbGVhc3Qgc3F1YXJlcwoKCmBgYHtyfQojIEJ1aWxkIHRoZSBtb2RlbCBvbiB0cmFpbmluZyBzZXQKc2V0LnNlZWQoMTIzKQptb2RlbC5wbHMgPC0gdHJhaW4oCiAgbWVkdn4uLCBkYXRhID0gdHJhaW4uYm9zdG9uLmRhdGEsIG1ldGhvZCA9ICJwbHMiLAogIHNjYWxlID0gVFJVRSwKICB0ckNvbnRyb2wgPSB0cmFpbkNvbnRyb2woImN2IiwgbnVtYmVyID0gMTApLAogIHR1bmVMZW5ndGggPSAxMAopCgojIFBsb3QgbW9kZWwgUk1TRSB2cyBEaWZmZXJlbnQgdmFsdWVzIG9mIGNvbXBvbmVudHMKcGxvdChtb2RlbC5wbHMpCmBgYAoKCmBgYHtyfQptb2RlbC5wbHMkYmVzdFR1bmUKYGBgCgpgYGB7cn0Kc3VtbWFyeShtb2RlbC5wbHMkZmluYWxNb2RlbCkKYGBgCgpgYGB7cn0KI01ha2UgcHJlZGljdGlvbnMKcHJlZGljdGlvbnMucGxzIDwtIG1vZGVsLnBscyAlPiUgcHJlZGljdCh0ZXN0LmJvc3Rvbi5kYXRhKQoKIyBNb2RlbCBwZXJmb3JtYW5jZSBtZXRyaWNzCgpkYXRhLmZyYW1lKAogIFJNU0UgPSBjYXJldDo6Uk1TRShwcmVkaWN0aW9ucy5wbHMsIHRlc3QuYm9zdG9uLmRhdGEkbWVkdiksCiAgUnNxdWFyZSA9IGNhcmV0OjpSMihwcmVkaWN0aW9ucy5wbHMsIHRlc3QuYm9zdG9uLmRhdGEkbWVkdikKKQpgYGAKCgpUaGUgb3B0aW1hbCBudW1iZXIgb2YgcHJpbmNpcGFsIGNvbXBvbmVudHMgaW5jbHVkZWQgaW4gdGhlIFBMUyBtb2RlbCBpcyA5LiBUaGlzIGNhcHR1cmVzIDkwJSBvZiB0aGUgdmFyaWF0aW9uIGluIHRoZSBwcmVkaWN0b3JzIGFuZCA3NSUgb2YgdGhlIHZhcmlhdGlvbiBpbiB0aGUgb3V0Y29tZSB2YXJpYWJsZSAobWVkdikuCgpJbiBvdXIgZXhhbXBsZSwgdGhlIGNyb3NzLXZhbGlkYXRpb24gZXJyb3IgUk1TRSBvYnRhaW5lZCB3aXRoIHRoZSBQTFMgbW9kZWwgaXMgbG93ZXIgdGhhbiB0aGUgUk1TRSBvYnRhaW5lZCB1c2luZyB0aGUgUENSIG1ldGhvZC4gU28sIHRoZSBQTFMgbW9kZWwgaXMgdGhlIGJlc3QgbW9kZWwsIGZvciBleHBsYWluaW5nIG91ciBkYXRhLCBjb21wYWVkIHRvIHRoZSBQQ1IgbW9kZWwuCgoKCgoKIyBDbGFzc2lmaWNhdGlvbgoKSW4gdGhpcyBwYXJ0LCB3ZSdsbCBjb212ZXIgdGhlIGZvbGx3aW5nIHRvcGljczoKCiogTG9naXN0aWMgcmVncmVzc2lvbiwgZm9yIGJpbmFyeSBjbGFzc2lmaWNhdGlvbiB0YXNrcwoqIFN0ZXB3aXNlIGFuZCBwZW5hbGl6ZWQgbG9naXN0aWMgcmVncmVzc2lvbiBmb3IgdmFpYWJsZSBzZWxlY3Rpb25zCiogTG9naXN0aWMgcmVncmVzc2lvbiBhc3N1bXB0aW9ucyBhbmQgZGlhZ25vc3RpY3MKKiBNdWx0aW5vbWlhbCBsb2dpc3RpYyByZWdyZXNzaW9uLCBhbmQgZXh0ZW5zaW9uIG9mIHRoZSBsb2dpc3RpYyByZWdyZXNzaW9uIGZvciBtdWx0aWNsYXNzIGNsYXNzaWZpY2F0aW9uIHRhc2tzCiogRGlzY3JpbWluYW50IGFuYWx5c2lzLCBmb3IgYmluYXJ5IGFuZCBtdWx0aWNsYXNzIGNsYXNzaWZpY2F0aW9uIHByb2JsZW1zCiogTmFpdmUgYmF5c2UgY2xhc3NpZmllcgoqIFN1cHBwb3J0IHZlY3RvciBtYWNoaW5lcwoqIENsYXNzaWZpY2F0aW9uIG1vZGVsIGV2YWx1YXRpb24KCk1vc3Qgb2YgdGhlIGNsYXNzaWZpY2F0aW9uIGFsZ29yaXRobXMgY29tcHV0ZXMgdGhlIHByb2JhYmlsaXR5IG9mIGJlbG9uZ2luZyB0byBhIGdpdmVuIGNsYXNzLgoKT2JzZXJ2YXRpb25zIGFyZSB0aGVuIGFzc2lnbmVkIHRvIHRoZSBjbGFzcyB0aGF0IGhhdmUgdGhlIGhpZ2hlc3QgcHJvYmFiaWxpdHkgc2NvcmUuCgpHZW5lcmFseSwgeW91IG5lZWQgdG8gZGVjaWRlIGEgcHJvYmFiaWxpdHkgY3V0b2ZmIGFib3ZlIHdoaWNoIHlvdSBjb25zaWRlciB0aGUgYW4gb2JzZXJ2YXRpb24gYXMgYmVsb25naW5nIHRvIGEgZ2l2ZW4gY2xhc3MuCgoKRGF0YXNldCB3aWxsIGJlIHVzaW5nIAoKMS4gUGltYUluZGlhbnNEaWFiZXRlczIgZGF0YSBzZXQKClRoZSBQaW1hIEluZGlhbiBEaWFiZXRlcyBkYXRhIHNldCBpcyBhdmFpbGFibGUgaW4gdGhlIG1sYmVuY2ggcGFja2FnZS4gSXQgd2lsbCBiZSB1c2VkIGZvciBiaW5hcnkgY2xhc3NpZmljYXRpb24uCgpgYGB7cn0KIyBMb2FkIHRoZSBkYXRhIHNldApkYXRhKCJQaW1hSW5kaWFuc0RpYWJldGVzMiIsIHBhY2thZ2UgPSAibWxiZW5jaCIpCgojIEluc3BlY3QgdGhlIGRhdGEKaGVhZChQaW1hSW5kaWFuc0RpYWJldGVzMiw0KQpgYGAKCmBgYHtyfQpzdHIoUGltYUluZGlhbnNEaWFiZXRlczIpCmBgYAoKVGhlIGRhdGEgY29udGFpbnMgNzY4IGluZGl2aWR1YWxzIChmZW1hbGUpIGFuZCA5IGNsaW5pY2FsIHZhcmlhYmxlcyBmb3IgcHJlZGljdGluIHRoZSBwcm9iYWJpbGl0eSBvZiBpbmRpdmlkdWFscyBpbiBiZWluZyBkaWFiZXRlLXBvc2l0aXZlIG9yIG5lZ2F0aXZlOgoKfCBDb2x1bW4gTmFtZXMgfCBEZXNjcmlwdGlvbiB8CnwgLS0tLS0tLS0tLS0tIHwgLS0tLS0tLS0tLS0gfAp8IHByZWduYW50IHwgCU51bWJlciBvZiB0aW1lcyBwcmVnbmFudCB8CnwgZ2x1Y29zZSB8IAlQbGFzbWEgZ2x1Y29zZSBjb25jZW50cmF0aW9uIChnbHVjb3NlIHRvbGVyYW5jZSB0ZXN0KSB8CnwgcHJlc3N1cmUgfCAJRGlhc3RvbGljIGJsb29kIHByZXNzdXJlIChtbSBIZykgfAp8IHRyaWNlcHMgfCAJVHJpY2VwcyBza2luIGZvbGQgdGhpY2tuZXNzIChtbSkgfAp8IGluc3VsaW4gfCAJMi1Ib3VyIHNlcnVtIGluc3VsaW4gKG11IFUvbWwpIHwKfCBtYXNzIHwgCUJvZHkgbWFzcyBpbmRleCAod2VpZ2h0IGluIGtnLyhoZWlnaHQgaW4gbSlcXjIpIHwKfCBwZWRpZ3JlZSB8IAlEaWFiZXRlcyBwZWRpZ3JlZSBmdW5jdGlvbiB8CnwgYWdlIHwgCUFnZSAoeWVhcnMpIHwKfCBkaWFiZXRlcyB8IAlDbGFzcyB2YXJpYWJsZSAodGVzdCBmb3IgZGlhYmV0ZXMpIHwKCgpQZXJmb3JtaW5nIHRoZSBmb2xsb3dpbmcgc3RlcHMgbWlnaHQgaW1wcm92ZSB0aGUgYWNjdXJhY3kgb2YgeW91ciBtb2RlbAoKKiBSZW1vdmUgcG90ZW50aWFsIG91dGxpZXJzCiogTWFrZSBzdXJlIHRoYXQgdGhlIHByZWRpY3RvciB2YXJpYWJsZXMgYXJlIG5vcm1hbGx5IGRpc3RyaWJ1dGVkLiBJZiBub3QsIHlvdSBjYW4gdXNlIGxvZywgcm9vdCwgQm94LUNveCB0cmFuc2Zvcm1hdGlvbi4KKiBSZW1vdmUgaGlnaGx5IGNvcnJlbGF0ZWQgcHJlZGljdG9ycyB0byBtaW5pbWl6ZSBvdmVyZml0dGluZy4gVGhlIHByZXNlbmNlIG9mIGhpZ2hseSBjb3JyZWxhdGVkIHByZWRpY3RvcnMgbWlnaHQgbGVhZCB0byBhbiB1bnN0YWJsZSBtb2RlbCBzb2x1dGlvbgoKYGBge3J9CnNldC5zZWVkKDEyMykKCnRyYWluaW5nLlBpbWEuc2FtcGxlcyA8LSBQaW1hSW5kaWFuc0RpYWJldGVzMiRkaWFiZXRlcyAlPiUKICBjcmVhdGVEYXRhUGFydGl0aW9uKHAgPSAwLjgsIGxpc3QgPSBGQUxTRSkKCnRyYWluLnBpbWEuZGF0YSA8LSBQaW1hSW5kaWFuc0RpYWJldGVzMlt0cmFpbmluZy5QaW1hLnNhbXBsZXMsIF0KdGVzdC5waW1hLmRhdGEgPC0gUGltYUluZGlhbnNEaWFiZXRlczJbLXRyYWluaW5nLlBpbWEuc2FtcGxlcywgXQpgYGAKCgojIyMgQ29tcHV0aW5nIGxvZ2lzdGljIHJlZ3Jlc3Npb24KCmBgYHtyfQojIEZpdCB0aGUgbW9kZWwKbW9kZWwubHIucGltYSA8LSBnbG0oZGlhYmV0ZXN+LiwgZGF0YSA9IHRyYWluLnBpbWEuZGF0YSwgZmFtaWx5ID0gYmlub21pYWwpCgojIFN1bW1hcml6ZSB0aGUgbW9kZWwKc3VtbWFyeShtb2RlbC5sci5waW1hKQoKIyMgTWFrZSBwcmVkaWN0aW9ucwoKcHJvYmFiaWxpdGllcyA8LSBtb2RlbC5sci5waW1hICU+JSBwcmVkaWN0KHRlc3QucGltYS5kYXRhLCB0eXBlID0gInJlc3BvbnNlIikKcHJlZGljdGVkLmNsYXNzZXMgPC0gaWZlbHNlKHByb2JhYmlsaXRpZXMgPiAwLjUsICJwb3MiLCAibmVnIikgCiMgcHJlZGljdGVkLmNsYXNzZXMKIyBNb2RlbCBhY2N1cmFjeQptZWFuKHByZWRpY3RlZC5jbGFzc2VzID09IHRlc3QucGltYS5kYXRhJGRpYWJldGVzKQpgYGAKCgoKIyMjIyBTaW1wbGUgbG9naXN0aWMgcmVncmVzc2lvbgoKVGhlIHNpbXBsZSBsb2dpc3RpYyByZWdyZXNzaW9uIGlzIHVzZWQgdG8gcHJlZGljdCB0aGUgcHJvYmFiaWxpdHkgb2YgY2xhc3MgbWVtYmVyc2hpcCBiYXNlZCBvbiBvbmUgc2luZ2xlIHByZWRpY3RvciB2YXJpYWJsZS4KClRoZSBmb2xsb3dpbmcgUiBjb2RlIGJ1aWxkcyBhIG1vZGVsIHRvIHByZWRpY3QgdGhlIHByb2JhYmlsaXR5IG9mIGJlaW5nICBkaWFiZXRlcy1wb3NpdGl2ZSBiYXNlZCBvbiB0aGUgcGxhc21hIGdsdWNvc2UgY29uY2VudHJhdGlvbjoKCmBgYHtyfQptb2RlbC5sb2dpdC5zaW1wbGUgPC0gZ2xtKGRpYWJldGVzIH4gZ2x1Y29zZSwgZGF0YSA9IHRyYWluLnBpbWEuZGF0YSwgZmFtaWx5ID0gYmlub21pYWwpCnN1bW1hcnkobW9kZWwubG9naXQuc2ltcGxlKQpgYGAKClRoZSBvdXRwdXQgYWJvdmUgc2hvd3MgdGhlIGVzdGltYXRlIG9mIHRoZSByZWdyZXNzaW9uIGJldGEgY29lZmZpY2llbnRzIGFuZCB0aGVpciBzaWduaWZpY2FuY2UgbGV2ZWxzLiBUaGUgaW50ZXJjZXB0IChiMCkgaXMgLTUuNTUgYW5kIHRoZSBjb2VmZmljaWVudCBvZiBnbHVjb3NlIHZhcmlhYmxlIGlzIDAuMDM5LgoKVGhlIGxvZ2lzdGljIGVxdWF0aW9uIGNhbiBiZSB3cml0dGVuIGFzIHAgPSBleHAoLTUuNTUgKyAwLjAzOSpnbHVjb3NlKS9bMStleHAoLTUuNTUrMC4wMzkqZ2x1Y29zZSldLiBVc2luZyB0aGlzIGZvcm11bGEsIGZvciBlYWNoIG5ldyBnbHVjb3NlIHBsYXNtYSBjb25jZW50cmF0aW9uIHZhbHVlLCB5b3UgY2FuIHByZWRpY3QgdGhlIHByb2JhYmlsaXR5IG9mIHRoZSBpbmRpdmlkdWFscyBpbiBiZWluIGRpYWJldGVzIHBvc2l0aXZlLgoKUHJlZGljdGlvbnMgY2FuIGJlIGVhc2lseSBtYWRlIHVzaW5nIHRoZSBmdW5jdGlvbiBwcmVkaWN0KCkuIFVzZSB0aGUgb3B0aW9uIHR5cGUgPSAicmVzcG9uc2UiIHRvIGRpcmVjdGx5IG9idGFpbiB0aGUgcHJvYmFiaWxpdGllcwoKYGBge3J9Cm5ld2RhdGEgPC0gZGF0YS5mcmFtZShnbHVjb3NlID0gYygyMCwxOTApKQpwcm9iYWJpbGl0aWVzIDwtIG1vZGVsLmxvZ2l0LnNpbXBsZSAgJT4lIHByZWRpY3QobmV3ZGF0YSwgdHlwZSA9ICJyZXNwb25zZSIpCnByZWRpY3RlZC5jbGFzc2VzIDwtIGlmZWxzZShwcm9iYWJpbGl0aWVzID4gMC41LCAicG9zIiwgIm5lZyIpCnByZWRpY3RlZC5jbGFzc2VzCmBgYAoKYGBge3J9CnRyYWluLnBpbWEuZGF0YSAlPiUKICBtdXRhdGUocHJvYiA9IGlmZWxzZShkaWFiZXRlcyA9PSAicG9zIiwgMSwgMCkpICU+JQogIGdncGxvdChhZXMoZ2x1Y29zZSxwcm9iKSkgKwogIGdlb21fcG9pbnQoYWxwaGEgPSAwLjIpICsKICBnZW9tX3Ntb290aChtZXRob2QgPSAiZ2xtIiwgbWV0aG9kLmFyZ3MgPSBsaXN0KGZhbWlseSA9ICJiaW5vbWlhbCIpKSsKICBsYWJzKAogICAgdGl0bGUgPSAiU2ltcGxlIExvZ2lzdGljIFJlZ3Jlc3Npb24gTW9kZWwiLAogICAgeCA9ICJQbGFzbWEgR2x1Y29zZSBDb25jZW50cmF0aW9uIiwKICAgIHkgPSAiUHJvYmFiaWxpdHkgb2YgYmVpbmcgZGlhYmV0ZXMtcG9zIgogICkKYGBgCgoKCmBgYHtyfQpwbG90KG1vZGVsLmxvZ2l0LnNpbXBsZSkKYGBgCgoKCgojIFN0ZXB3aXNlIExvZ2lzdGljIFJlZ3Jlc3Npb24KCioqU3RlcHdpc2UgbG9naXN0aWMgcmVncmVzc2lvbioqIGNvbnNpc3RzIG9mIGF1dG9tYXRpY2FsbHkgc2VsZWN0aW5nIGEgcmVkdWNlZCBudW1iZXIgb2YgcHJlZGljdG9yIHZhcmlhYmxlcyBmb3IgYnVpbGRpbmcgdGhlIGJlc3QgcGVyZm9ybWluZyBsb2dpc3RpYyByZWdyZXNzaW9uIG1vZGVsLgoKCkRhdGEgc2V0OiAqKlBpbWFJbmRpYW5zRGlhYmV0ZXMyKioKCgpRdWljayBzdGFydCBSIGNvZGUKCmBgYHtyfQpsaWJyYXJ5KE1BU1MpCiMgRml0IHRoZSBtb2RlbAoKcmVtb3ZlZC5taXNzaW5nLmRhdGEgPC0gbmEub21pdCh0cmFpbi5waW1hLmRhdGEpCm1vZGVsIDwtIGdsbShkaWFiZXRlcyB+IC4sIGRhdGEgPSByZW1vdmVkLm1pc3NpbmcuZGF0YSAsIGZhbWlseSA9IGJpbm9taWFsKSAlPiUgc3RlcEFJQyh0cmFjZSA9IEZBTFNFKQoKc3VtbWFyeShtb2RlbCkKCnJlbW92ZWQubWlzc2luZy5kYXRhLmZyb20udGVzdCA8LSBuYS5vbWl0KHRlc3QucGltYS5kYXRhKQoKcHJvYmFiaWxpdGllcy5zdGVwIDwtIG1vZGVsICU+JSBwcmVkaWN0KHJlbW92ZWQubWlzc2luZy5kYXRhLmZyb20udGVzdCwgdHlwZSA9ICJyZXNwb25zZSIpCnByZWRpY3RlZC5jbGFzc2VzLnN0ZXAubG9naXQgPC0gaWZlbHNlKHByb2JhYmlsaXRpZXMuc3RlcCA+IDAuNSwgInBvcyIsICJuZWciKQoKI01vZGVsIEFjY3VyYWN5Cm1lYW4ocHJlZGljdGVkLmNsYXNzZXMuc3RlcC5sb2dpdCA9PSByZW1vdmVkLm1pc3NpbmcuZGF0YS5mcm9tLnRlc3QkZGlhYmV0ZXMpCmBgYAoKYGBge3J9CmRmIDwtIGRhdGEuZnJhbWUoIlByZWRpY3RlZCIgPSBwcmVkaWN0ZWQuY2xhc3Nlcy5zdGVwLmxvZ2l0KQooZGYkUHJlZGljdGVkID09IHRlc3QucGltYS5kYXRhJGRpYWJldGVzKQpgYGAKCgoKIyMgUGVuYWxpemVkIGxvZ2lzdGljIHJlZ3Jlc3Npb24KCioqUGVuYWxpemVkIGxvZ2lzdGljIHJlZ3Jlc3Npb24qKiBpbXBvc2VzIGEgcGVuYWx0eSB0byB0aGUgbG9naXN0aWMgbW9kZWwgZm9yIGhhdmluZyB0b28gbWFueSB2YXJpYWJsZXMuIFRoaXMgcmVzdWx0cyBpbiBzaHJpbmtpbmcgdGhlIGNvZWZmaWNpZW50cyBvZiB0aGUgbGVzcyBjb250cmlidXRpdmUgdmFyaWFibGVzIHRvd2FyZCB6ZXJvLiBUaGlzIGlzIGFsc28ga25vdyBhcyAqKnJlZ3VsYXJpemF0aW9uKiouClRoZSBtb3N0IGNvbW1vbmx5IHVzZWQgcGVuYWxpemVkIHJlZ3Jlc3Npb24gaW5jbHVkZQoKKiAqKlJpZGdlIHJlZ3Jlc3Npb24qKjogdmFyaWFibGVzIHdpdGggbWlub3IgY29udHJpYnV0aW9uIGhhdmUgdGhlaXIgY29lZmZpY2llbnRzIGNsb3NlIHRvIHplcm8uCkhvd2V2ZXIsIGFsbCB0aGUgdmFyaWFibGVzIGFyZSBpbmNvcnBvcmF0ZWQgaW4gdGhlIG1vZGVsLiBUaGlzIGlzIHVzZWZ1bCB3aGVuIGFsbCB2YXJpYWJsZXMgbmVlZCB0byBiZSBpbmNvcnBvcmF0ZWQgaW4gdGhlIG1vZGVsIGFjY29yZGluZyB0byBkb21haW4ga25vd2xlZGdlLgoKKiAqKmxhc3NvIHJlZ3Jlc3Npb24qKjogdGhlIGNvZWZmaWNpZW50cyBvZiBzb21lIGxlc3MgY29udHJpYnV0aXZlIHZhcmlhYmxlcyBhZXIgZm9yY2VkIHRvIGJlIGV4YWN0bHkgemVyby4gT25seSB0aGUgbW9zdCBzaWduaWZpY2FudCB2YXJpYWJsZXMgYWUga2VwdCBpbiB0aGUgZmluYWwgbW9kZWwuCgoqICoqZWxhc3RpYyBuZXQgcmVncmVzc2lvbioqOiB0aGUgY29tYmluYXRpb24gb2YgcmlkZ2UgYW5kIGxhc3NvIHJlZ3Jlc3Npb24uIEl0IHNocmlua3Mgc29tZSBjb2VmZmljaWVudHMgdG93YXJkIHplcm8gKGxpa2UgcmlkZ2UgcmVnZXJzc2lvbikgYW5kIHNldCBzb21lIGNvZWZmaWNpZW50cyB0byBleGFjdGx5IHplcm8gKGxpa2UgbGFzc28gcmVncmVzc2lvbikKCgpSZXF1aXJlZCBwYWNrYWdlcwoKYGBge3J9CmxpYnJhcnkodGlkeXZlcnNlKQpsaWJyYXJ5KGNhcmV0KQpsaWJyYXJ5KGdsbW5ldCkKYGBgCgpQcmVwYXJpbmcgdGhlIGRhdGEKCkRhdGEgc2V0OiBQaW1hSW5kaWFuc0RpYWJldGVzMgoKYGBge3J9CiMgTG9hZCB0aGUgZGF0YSBhbmQgcmVtb3ZlIE5BcwpQaW1hLkluZGlhbnMuZGF0YSA8LSBuYS5vbWl0KFBpbWFJbmRpYW5zRGlhYmV0ZXMyKQojIEluc3BlY3QgdGhlIGRhdGEKc2FtcGxlX24oUGltYS5JbmRpYW5zLmRhdGEsIDMpCgojIFNwbGl0IHRoZSBkYXRhIGludG8gdHJhaW5pbmcgYW5kIHRlc3Qgc2V0CnNldC5zZWVkKDEyMykKCnRyYWluaW5nLnNhbXBsZXMgPC0gUGltYS5JbmRpYW5zLmRhdGEkZGlhYmV0ZXMgJT4lCiAgY3JlYXRlRGF0YVBhcnRpdGlvbihwID0gMC44LCBsaXN0ID0gRkFMU0UpCgp0cmFpbi5waW1hLmluZGlhbnMgPC0gUGltYS5JbmRpYW5zLmRhdGFbdHJhaW5pbmcuc2FtcGxlcywgXQp0ZXN0LnBpbWEuaW5kaWFucyA8LSBQaW1hLkluZGlhbnMuZGF0YVstdHJhaW5pbmcuc2FtcGxlcywgXQpgYGAKCkFkZGl0aW9uYWwgZGF0YSBwcmVwYXJhdGlvbgoKVGhlIFIgZnVuY3Rpb24gbW9kZWwubWF0cml4KCkgaGVscCB0byBjcmVhdGUgdGhlIG1hdHJpeCBvZiBwcmVkaWN0b3JzIGFuZCBhbHNvIGF1dG9tYXRpY2FsbHkgY29udmVydHMgY2F0ZWdvcmljYWwgcHJlZGljdG9ycyB0byBhcHByb3ByaWF0ZSBkdW1teSB2YXJpYWJsZXMsIHdoaWNoIGlzIHJlcXVpcmVkIGZvciB0aGUgZ2xtbmV0KCkgZnVuY3Rpb24uCgpgYGB7cn0KIyBEdW1teSBjb2RlIGNhdGVnb3JpY2FsIHByZWRpY3RvciB2YXJpYWJsZXMKeCA8LSBtb2RlbC5tYXRyaXgoZGlhYmV0ZXMgfi4sIHRyYWluLnBpbWEuaW5kaWFucylbLC0xXQoKIyBDb252ZXJ0IHRoZSBvdXRjb21lIChjbGFzcykgdG8gYSBudW1lcmljYWwgdmFyaWFibGUKeSA8LSBpZmVsc2UodHJhaW4ucGltYS5pbmRpYW5zJGRpYWJldGVzID09ICJwb3MiLCAxLCAwKQpgYGAKCgoKV2UnbGwgdXNlIHRoZSBSIGZ1bmN0aW9uIGdsbW5ldCgpIGZvciBjb21wdXRpbmcgcGVuYWxpemVkIGxvZ2lzdGljIHJlZ3Jlc3Npb24uCgpUaGUgc2ltcGxpZmllZCBmb3JtYXQgaXMgYXMgZm9sbG93OgoKCmBgYHtyfQpsaWJyYXJ5KGdsbW5ldCkKZ2xtbmV0KHgsIHksIGZhbWlseSA9ICJiaW5vbWlhbCIsIGFscGhhID0gMSwgbGFtYmRhID0gTlVMTCkKYGBgCgoKYGBge3J9CnNldC5zZWVkKDEyMykKY3YubGFzc28gPC0gY3YuZ2xtbmV0KHgseSwgYWxwaGEgPSAxLCBmYW1pbHkgPSAiYmlub21pYWwiKQojIGN2Lmxhc3NvCiMgc3VtbWFyeShjdi5sYXNzbykKIyBGaXQgdGhlIGZpbmFsIG1vZGVsIG9uIHRoZSB0cmFpbmluZyBkYXRhCm1vZGVsLmxvZ2l0Lmxhc3NvIDwtIGdsbW5ldCh4LHksIGFscGhhID0gMSwgZmFtaWx5ID0gImJpbm9taWFsIiwgbGFtYmRhID0gY3YubGFzc28kbGFtYmRhLm1pbikKCiMgRGlzcGxheSByZWdyZXNzaW9uIGNvZWZmaWNpZW50cwpjb2VmKG1vZGVsLmxvZ2l0Lmxhc3NvKQoKIyBNYWtlIHByZWRpY3Rpb25zIG9uIHRoZSB0ZXN0IGRhdGEKeC50ZXN0IDwtIG1vZGVsLm1hdHJpeChkaWFiZXRlc34uLCB0ZXN0LnBpbWEuaW5kaWFucylbLC0xXQpwcm9iYWJpbGl0aWVzLmxvZ2l0Lmxhc3NvIDwtIG1vZGVsLmxvZ2l0Lmxhc3NvICU+JSBwcmVkaWN0KG5ld3ggPSB4LnRlc3QpCnByZWRpY3QuY2xhc3Nlc3MubG9naXQubGFzc28gPC0gaWZlbHNlKHByb2JhYmlsaXRpZXMubG9naXQubGFzc28gPiAwLjUsICJwb3MiLCAibmVnIikKCiNNb2RlbCBhY2N1cmFjeQpvYnNlcnZlZC5jbGFzc2VzIDwtIHRlc3QucGltYS5pbmRpYW5zJGRpYWJldGVzCm1lYW4ocHJlZGljdC5jbGFzc2Vzcy5sb2dpdC5sYXNzbyA9PSBvYnNlcnZlZC5jbGFzc2VzKQpgYGAKCmBgYHtyfQpwbG90KGN2Lmxhc3NvKQpgYGAKCgpUaGUgcGxvdCBkaXNwbGF5cyB0aGUgY3Jvc3MtdmFsaWRhdGlvbiBlcnJvciBhY2NvcmRpbmcgdG8gdGhlIGxvZyBvZiBsYW1iZGEuIFRoZSBsZWZ0IGRhc2hlZCB2ZXJ0aWNhbCBsaW5lIGluZGljYXRlcyB0aGF0IHRoZSBsb2cgb2YgdGhlIG9wdGltYWwgdmFsdWUgb2YgbGFtYmRhIGlzIGFwcHJveGltYXRlbHkgLTUsIHdoaWNoIGlzIHRoZSBvbmUgdGhhdCBtaW5pbWl6ZXMgdGhlIHByZWRpY3Rpb24gZXJyb3IuIFRoaXMgbGFtYmRhIHZhbHVlIHdpbGwgZ2l2ZSB0aGUgbW9zdCBhY2N1cmF0ZSBtb2RlbC4gVGhlIGV4YWN0IHZhbHVlIG9mIGxhbWJkYSBjYW4gYmUgdmlld2VkIGFzIGZvbGxvdzoKCmBgYHtyfQpjdi5sYXNzbyRsYW1iZGEubWluCmN2Lmxhc3NvJGxhbWJkYS4xc2UKYGBgCgpVc2luZyBsYW1iZGEubWluIGFzIHRoZSBiZXN0IGxhbWJkYSwgZ2l2ZXMgdGhlIGZvbGxvd2luZyByZWdyZXNzaW9uIGNvZWZmaWNpZW50czoKYGBge3J9CmNvZWYoY3YubGFzc28sIGN2Lmxhc3NvJGxhbWJkYS5taW4pCmBgYAoKYGBge3J9CmNvZWYoY3YubGFzc28sIGN2Lmxhc3NvJGxhbWJkYS4xc2UpCmBgYAoKCgojIyMgTG9naXN0aWMgUmVncmVzc2lvbiBBc3N1bXB0aW9ucyBhbmQgRGlhZ25vc3RpY3MKCgpUaGUgbG9naXN0aWMgcmVncmVzc2lvbiBtZXRob2QgYXNzdW1lcyB0aGF0OgoKKiBUaGUgb3V0Y29tZSBpcyBhIGJpbmFyeSBvciBkaWNob3RvbW91cyB2YXJpYWJsZSBsaWtlIHllcyB2cyBubywgcG9zaXRpdmUgdnMgbmVnYXRpdmUsIDEgdnMgMC4KKiB0aGVyZSBpcyBhbGluZWFyZSByZWxhdGlvbnNoaXAgYmV0d2VlbiB0aGUgbG9naXQgb2YgdGhlIG91dGNvbWUgYW5kIGVhY2ggcHJlZGljdG9yIHZhcmlhYmxlcy4gUmVjYWxsIHRoYXQgdGhlIG91dGNvbWUgYW5kIGVhY2ggcHJlZGljdG9yIHZhcmlhYmxlcy4gUmVjYWxsIHRoYXQgdGhlIGxvZ2l0IGZ1bmN0aW9uIGlzIGxvZ2l0KHApID0gbG9nKHAvKDEtcCkpLCB3aGVyZSBwIGlzIHRoZSBwcm9iYWJpbGl0aWVzIG9mIHRoZSBvdXRjb21lCgoqIFRoZXJlIGlzIG5vIGluZmx1ZW50aWFsIHZhbHVlIChleHRyZWFtZSB2YWx1ZXMgb3Igb3V0bGllcnMpIGluIHRoZSBjb250aW51b3VzIHByZWRpY3RvcnMKKiBUaGVyZSBpcyBubyBoaWdoIGludGVyY29ycmVsYXRpb24gKGkuZS4gbXVsdGljb2xsaW5lYXJpdHkpIGFtb25nIHRoZSBwcmVkaWN0b3JzLgoKTG9hZGluZyByZWVxdWlyZWQgUiBwYWNrYWdlcwoKYGBge3J9CmxpYnJhcnkodGlkeXZlcnNlKQpsaWJyYXJ5KGJyb29tKQp0aGVtZV9zZXQodGhlbWVfY2xhc3NpYygpKQpgYGAKCmBgYHtyfQpQaW1hSW5kaWFuc0RpYWJldGVzMi5jbGVhbmVkIDwtIG5hLm9taXQoUGltYUluZGlhbnNEaWFiZXRlczIpCiMgRml0IHRoZSBsb2dpc3RpYyByZWdyZXNzaW9uIG1vZGVsCm1vZGVsLnBpbWEubG9naXQyIDwtIGdsbShkaWFiZXRlc34uLCBkYXRhID0gUGltYUluZGlhbnNEaWFiZXRlczIuY2xlYW5lZCwgZmFtaWx5ID0gYmlub21pYWwpCiMgUHJlZGljdCB0aGUgcHJvYmFiaWxpdHkgKHApIG9mIGRpaWFiZXRlIHBvc2l0aXZpdHkKcHJvYmFiaWxpdGllcy5waW1hLmxvZ2l0IDwtIHByZWRpY3QobW9kZWwucGltYS5sb2dpdDIsIHR5cGUgPSAicmVzcG9uc2UiKQpwcmVkaWN0ZWQucGltYS5jbGFzc2VzIDwtIGlmZWxzZShwcm9iYWJpbGl0aWVzLnBpbWEubG9naXQgPiAwLjUsICJwb3MiLCAibmVnIikKaGVhZChwcmVkaWN0ZWQucGltYS5jbGFzc2VzKQpgYGAKCgpgYGB7cn0KIyB0cmFpbi5waW1hLmluZGlhbnMKbXlkYXRhIDwtIFBpbWFJbmRpYW5zRGlhYmV0ZXMyLmNsZWFuZWQgICU+JQogIGRwbHlyOjpzZWxlY3RfaWYoaXMubnVtZXJpYykKcHJlZGljdG9ycyA8LSBjb2xuYW1lcyhteWRhdGEpCiMgQmluZCB0aGUgbG9naXQgYW5kIHRpZHlpbmcgdGhlIGRhdGEgZm9yIHBsb3QKbW9kZWwucGltYS5sb2dpdCA8LSBteWRhdGEgJT4lCiAgbXV0YXRlKGxvZ2l0ID0gbG9nKHByb2JhYmlsaXRpZXMucGltYS5sb2dpdC8oMS1wcm9iYWJpbGl0aWVzLnBpbWEubG9naXQpKSkgJT4lCiAgZ2F0aGVyKGtleSA9ICJwcmVkaWN0b3JzIiwgdmFsdWUgPSAicHJlZGljdG9yLnZhbHVlIiwgLWxvZ2l0KQpgYGAKYGBge3J9CmdncGxvdChtb2RlbC5waW1hLmxvZ2l0LCBhZXMobG9naXQsIHByZWRpY3Rvci52YWx1ZSkpICsKICBnZW9tX3BvaW50KHNpemUgPSAwLjUsIGFscGhhID0gMC41KSArCiAgZ2VvbV9zbW9vdGgobWV0aG9kID0gImxvZXNzIikgKwogIHRoZW1lX2J3KCkgKwogIGZhY2V0X3dyYXAofnByZWRpY3RvcnMsIHNjYWxlcyA9ICJmcmVlX3kiKQpgYGAKCiMjIyBJbmZsdWVudGlhbCB2YWx1ZXMKCkluZmx1ZW50aWFsIHZhbHVlcyBhcmUgZXh0cmVtZSBpbmRpdmlkdWFsIGRhdGEgcG9pbnRzIHRoYXQgY2FuIGFsdGVyIHRoZSBxdWFpdHkgb2YgdGhlIGxvZ2lzdGljIHJlZ3Jlc3Npb24gbW9kZWwuClRoZSBtb3N0IGV4dHJlbWUgdmFsdWVzIGluIHRoZSBkYXRhIGNhbiBiZSBleGFtaW5lZCBieSB2aXN1YWxpemluZyB0aGUgQ29vaydzIGRpc3RhbmNlIHZhbHVlcy4KCkhlcmUgd2UgbGFiZWwgdGhlIHRvcCAzIGxhcmdlc3QgdmFsdWVzOgoKYGBge3J9CnBsb3QobW9kZWwucGltYS5sb2dpdDIsIHdoaWNoID0gNCwgaWQubiA9IDMpCmBgYAoKTm90ZSB0aGF0LCBub3QgYWxsIG91dGxpZXJzIGFyZSBpbmZsdWVudGlhbCBvYnNlcnZhdGlvbnMuIFRvIGNoZWNrIGt3aGV0aGVyIHRoZSBkYXRhIGNvbnRhaW5zIHBvdGVudGlhbCBpbmZsdWVudGlhbCBvYnNlcnZhdGlvbnMsIHRoZSBzdGFuZGFyZGl6ZWQgcmVzaWR1YWwgZXJyb3IgY2FuIGJlIGluc3BlY3RlZC4gRGF0YSBwb2ludHMgd2l0aCBhbiBhYnNvbHV0ZSBzdGFuZGFyZGl6ZWQgcmVzaWR1YWxzIGFib3ZlIDMgcmVwcmVzZW50IHBvc3NpYmxlIG91dGxpZXJzIGFuZCBtYXkgZGVzZXJ2ZSBjbG9zZXIgYXR0ZW50aW9uLgoKVGhlIGZvbGxvd2luZyBSIGNvZGUgY29tcHV0ZXMgdGhlIHN0YW5kYXJkaXplZCByZXNpZHVhbHMgKC5zdGQucmVzaWQpIGFuZCB0aGUgQ29vaydzIGRpc3RhbmNlICguY29va3NkKSB1c2luZyB0aGUgUiBmdW5jdGlvbiBhdWdtZW50KClbYnJvb20gcGFja2FnZV0uCgoKYGBge3J9CiMgRXh0cmFjdCBtb2RlbCByZXN1bHRzCgptb2RlbC5waW1hLmxvZ2l0LmRhdGEgPC0gYXVnbWVudChtb2RlbC5waW1hLmxvZ2l0MikgJT4lCiAgbXV0YXRlKGluZGV4ID0gMTpuKCkpCmBgYAoKVGhlIGRhdGEgZm9yIHRoZSB0b3AgMyBsYXJnZXN0IHZhbHVlcywgYWNjb3JkaW5nIHRvIHRoZSBDb29rJ3MgZGlzdGFuY2UsIGNhbiBiZSBkaXNwbGF5ZWQgYXMgZm9sbG93OgoKYGBge3J9Cm1vZGVsLnBpbWEubG9naXQuZGF0YSAlPiUgdG9wX24oMywgLmNvb2tzZCkKCmBgYAoKUGxvdCB0aGUgc3RhbmRhcmRpemVkIHJlc2lkdWFsczoKCmBgYHtyfQpnZ3Bsb3QobW9kZWwucGltYS5sb2dpdC5kYXRhLCBhZXMoaW5kZXgsIC5zdGQucmVzaWQpKSArCiAgZ2VvbV9wb2ludChhZXMoY29sb3IgPSBkaWFiZXRlcyksIGFscGhhID0gMC41KSArCiAgdGhlbWVfYncoKQpgYGAKCgpGaWx0ZXIgcG90ZW50aWFsIGluZmx1ZW50aWFsIGRhdGEgcG9pbnRzIHdpdGggYWJzKC5zdGQucmVzKSA+IDMKCmBgYHtyfQptb2RlbC5waW1hLmxvZ2l0LmRhdGEgJT4lIGZpbHRlcihhYnMoLnN0ZC5yZXNpZCkgPiAzKQpgYGAKPiBUaGVyZSBpcyBubyBpbmZsdWVudGlhbCBvYnNlcnZhdGlvbiBpbiBvdXIgZGF0YS4KCldoZW4gd2UgaGF2ZSBvdXRsaWVycyBpbiBhIGNvbnRpbnVvdXMgcHJlZGljdG9yLCBwb3RlbnRpYWwgc29sdXRpb25zIGluY2x1ZGU6CgoqIFJlbW92aW5nIHRoZSBjb25jZXJuZWQgcmVjb3JkcwoqIFRyYW5zZm9ybSB0aGUgZGF0YSBpbnRvIGxvZyBzY2FsZQoqIFVzZSBub24gcGFyYW1ldGVyaWMgbWV0aG9kcwoKIyMjIE11bHRpY29sbGluZWFyaXR5CgpNdWx0aWNvbGxpbmVhcml0eSBjb3JyZXNwb25kcyB0byBhIHNpdHVhdGlvbiB3aGVyZSB0aGUgZGF0YSBjb250YWluIGhpZ2hseSBjb3JyZWxhdGVkIHByZWRpY3RvciB2YXJpYWJsZXMuCgpNdWx0aWNvbGxpbmVhcml0eSBpcyBhbiBpbXBvcnRhbnQgaXNzdWUgaW4gcmVncmVzc2lvbiBhbmFseXNpcyBhbmQgc2hvdWxkIGJlIGZpeGVkIGJ5IHJlbW92aW5nIHRoZSBjb25jZXJuZWQgdmFyaWFibGVzLiBJdCBjYW4gYmUgYXNzZXNzZWQgdXNpbmcgdGhlIFIgZnVuY3Rpb24gdmlmKCkKCmBgYHtyfQpjYXI6OnZpZihtb2RlbC5waW1hLmxvZ2l0MikKYGBgCgpBcyBhIHJ1bGUgb2YgdGh1bWIsIGEgVklGIHZhbHVlIHRoYXQgZXhjZWVkcyA1IG9yIDEwIGluZGljYXRlcyBhIHByb2JsZW1hdGljIGFtb3VudCBvZiBjb2xsaW5lYXJpdHkuIEluIG91ciBleGFtcGxlIHRoZXJlIGlzIG5vIGNvbGxpbmVhcml0eTogYWxsIHZhcmlhYmxlcyBoYXZlIGEgdmFsdWUgb2YgVklGIHdlbGwgYmVsb3cgNS4KCgojIyBNdWx0aW5vbWlhbCBsb2dpc3RpYyByZWdyZXNzaW9uCgpUaGUgKiptdWx0aW5vbWlhbCBsb2dpc3RpYyByZWdyZXNzaW4qKiBpcyBhbiBleHRlbnNpb24gb2YgdGhlIGxvZ2lzdGljIHJlZ3Jlc3Npb24gZm9yIG11bHRjbGFzcyBjbGFzc2lmY2F0aW9uIHRhc2tzLiBJdCBpcyB1c2VkIHdoZW4gdGhlIG91dGNvbWUgaW52b2x2ZXMgbW9yZSB0aGFuIHR3byBjbGFzc2VzLgoKTG9hZGluZyByZXF1aXJlZCBSIHBhY2thZ2VzCmBgYHtyfQpsaWJyYXJ5KHRpZHl2ZXJzZSkKbGlicmFyeShjYXJldCkKbGlicmFyeShubmV0KQpgYGAKCiMjIyBQcmVwYXJpbmcgdGhlIGRhdGEKCldlJ2xsIHVzZSB0aGUgaXJpcyBkYXRhIHNldAoKYGBge3J9CiMgTG9hZCB0aGUgZGF0YQpkYXRhKCJpcmlzIikKIyBJbnNwZWN0IHRoZSBkYXRhCnNhbXBsZV9uKGlyaXMsMykKCiMgU3BsaXQgdGhlIGRhdGEgaW50byB0cmFpbmluZyBhbmQgdGVzdCBzZXQKc2V0LnNlZWQoMTIzKQp0cmFpbmluZy5zYW1wbGVzLmlyaXMgPC0gaXJpcyRTcGVjaWVzICU+JSBjcmVhdGVEYXRhUGFydGl0aW9uKHA9MC44LCBsaXN0ID0gRkFMU0UpCnRyYWluLmlyaXMuZGF0YSA8LSBpcmlzW3RyYWluaW5nLnNhbXBsZXMuaXJpcyxdCnRlc3QuaXJpcy5kYXRhIDwtIGlyaXNbLXRyYWluaW5nLnNhbXBsZXMuaXJpcyxdCmBgYAoKCiMjIyBDb21wdXRpbmcgbXVsdGlub21pYWwgbG9naXN0aWMgcmVncmVzc2lvbgpgYGB7cn0KIyBGaXQgdGhlIG1vZGVsCm1vZGVsIDwtIG5uZXQ6Om11bHRpbm9tKFNwZWNpZXMgfi4sIGRhdGEgPSB0cmFpbi5pcmlzLmRhdGEpCiMgU3VtbWFyaXplIHRoZSBtb2RlbApzdW1tYXJ5KG1vZGVsKQoKIyBNYWtlIHByZWRpY3Rpb25zCgpwcmVkaWN0ZWQuc3BlY2llcyA8LSBtb2RlbCAlPiUgcHJlZGljdCh0ZXN0LmlyaXMuZGF0YSkKaGVhZChwcmVkaWN0ZWQuc3BlY2llcykKIyBNb2RlbCBhY2N1cmFjeQptZWFuKHByZWRpY3RlZC5zcGVjaWVzID09IHRlc3QuaXJpcy5kYXRhJFNwZWNpZXMpCmBgYAoKIyMgRGlzY3JpbWluYW50IEFuYWx5c2lzCgpUaGUgZm9sbG93aW5nIGRpc2NyaW1pbmFudCBhbmFseXNpcyBtZXRob2RzIHdpbGwgYmUgZGVzY3JpYmVkOgoKKiBMaW5lYXIgZGlzY3JpbWluYW50IGFuYWx5c2lzIChMREEpOiBVc2VzIGxpbmVhciBjb21iaW5hdGlvbnMgb2YgcHJlZGljdG9ycyB0byBwcmVkaWN0IHRoZSBjbGFzcyBvZiBhIGdpdmVuIG9ic2VydmF0aW9uLiBBc3N1bWVzIHRoYXQgdGhlIHByZWRpY3RvciB2YXJpYWJsZXMgKHApIGFyZSBub3JtYWxseSBkaXN0cmlidXRlZCBhbmQgdGhlIGNsYXNzZXMgaGF2ZSBpZGVudGljYWwgdmFyaWFuY2VzIChmb3IgdW5pdmFyaWF0ZSBhbmFseXNpcywgcCA9IDEpIG9yIGlkZW50aWNhbCBjb3ZhcmlhbmNlIG1hdHJpY2VzIChmb3IgbXVsdHZhcmlhdGUgYW5hbHlzaXMsIHAgPiAxKS4KCiogUXVhZHJhdGljIGRpc2NyaW1pbmFudCBhbmFseXNpcyAoUURBKTogTW9yZSBmbGV4aWJsZSB0aGFuIExEQS4gSGVyZSwgdGhlcmUgaXMgbm8gYXNzdW1wdGlvbiB0aGF0IHRoZSBjb3ZhcmlhbmNlIG1hdHJpeCBvZiBjbGFzc2VzIGlzIHRoZSBzYW1lCgoqIE1peHR1cmUgZGlzY3JpbWluYW50IGFuYWx5c2lzIChNREEpOiBFYWNoIGNsYXNzIGlzIGFzc3VtZWQgdG8gYmUgYSBHYXVzc2lhbiBtaXh0dXJlIG9mIHN1YmNsYXNzZXMuCgoqIEZsZXhpYmxlIERpc2NyaW1pbmFudCBBbmFseXNpcyAoRkRBKTogTm9uLWxpbmVhciBjb21iaW5hdGlvbiBvZiBwcmVkaWN0b3JzIGlzIHVzZWQgc3VjaCBhcyBzcGxpbmVzLgoKKiBSZWd1bGFyaXplZCBkaXNjcmltaW5hbnQgYW5hbHlzaXMgKFJEQSk6IFJlZ3VsYXJpemF0aW9uIChvciBzaHJpbmthZ2UpIGltcHJvdmVzIHRoZSBlc3RpbWF0ZSBvZiB0aGUgY292YXJpYW5jZSBtYXRyaWNlcyBpbiBzaXR1YXRpb24gd2hlcmUgdGhlIG51bWJlciBvZiBwcmVkaWN0b3JzIGlzIGxhcmdlciB0aGFuIHRoZSBudW1iZXIgb2Ygc2FtcGxlcyBpbiB0aGUgdHJhaW5pbmcgZGF0YS4gVGhpcyBsZWFkcyB0byBhbiBpbXByb3ZlbWVudCBvZiB0aGUgZGlzY3JpbWluYW50IGFuYWx5c2lzLgoKIyMjIExvYWRpbmcgcmVxdWlyZWQgUiBwYWNrYWdlcwoKYGBge3J9CmxpYnJhcnkodGlkeXZlcnNlKQpsaWJyYXJ5KGNhcmV0KQp0aGVtZV9zZXQodGhlbWVfY2xhc3NpYygpKQpgYGAKCgojIyMgUHJlcGFyaW5nIHRoZSBkYXRhCjEuIFNwbGl0IHRoZSBkYXRhCgpgYGB7cn0KIyBMb2RkIHRoZSBkYXRhCmRhdGEoImlyaXMiKQojIFNwbGl0IHRoZSBkYXRhIGludG8gdHJhaW5pbmcgYW5kIHRlc3Qgc2V0CnNldC5zZWVkKDEyMykKdHJhaW5pbmcuc2FtcGxlcy5pcmlzIDwtIGlyaXMkU3BlY2llcyAlPiUKICBjcmVhdGVEYXRhUGFydGl0aW9uKHA9MC44LCBsaXN0ID0gRkFMU0UpCgp0cmFpbi5kYXRhLmlyaXMgPC0gaXJpc1t0cmFpbmluZy5zYW1wbGVzLmlyaXMsIF0KdGVzdC5kYXRhLmlyaXMgPC0gaXJpc1stdHJhaW5pbmcuc2FtcGxlcy5pcmlzLCBdCmBgYAoKMi4gTm9ybWFsaXplIHRoZSBkYXRhLiBDYXRlZ29yaWNhbCAKCmBgYHtyfQojIEVzdGltYXRlIHByZXByb2Nlc3NpbmcgcGFyYW1ldGVycwpwcmVwcm9jLnByYW0gPC0gdHJhaW4uZGF0YS5pcmlzICU+JQogIHByZVByb2Nlc3MobWV0aG9kID0gYygiY2VudGVyIiwgInNjYWxlIikpCgojIFRyYW5zZm9ybSB0aGUgZGF0YSB1c2luZyB0aGUgZXN0aW1hdGVkIHBhcmFtZXRlcnMKdHJhaW4udHJhbnNmb3JtZWQgPC0gcHJlcHJvYy5wcmFtICU+JSBwcmVkaWN0KHRyYWluLmRhdGEuaXJpcykKdGVzdC50cmFuc2Zvcm1lZCA8LSBwcmVwcm9jLnByYW0gJT4lIHByZWRpY3QodGVzdC5kYXRhLmlyaXMpCmBgYAoKCkJlZm9yZSBwZXJmb3JtaW5nIExEQSwgY29uc2lkZXI6CiogSW5zcGVjdGluZyB0aGUgdW5pdmFyaWF0ZSBkaXN0cmlidXRpb24gb2YgZWFjaCB2YXJpYWJsZSBhbmQgbWFrZSBzdXJlIHRoYXQgdGhleSBhcmUgbm9ybWFsbHkgZGlzdHJpYnV0ZS4gSWYgbm90LCB5b3UgY2FuIHRyYW5zZm9ybSB0aGVtIHVzaW5nIGxvZyBhbmQgcm9vdCBmb3IgZXhwb25lbnRpYWwgZGlzdHJpYnV0aW9ucyBhbmQgQm94LUNveCBmb3Igc2tld2VkIGRpc3RyaWJ1dGlvbnMuCgoqcmVtb3Zpbmcgb3V0bGllcnMgZnJvbSB5b3VyIGRhdGEgYW5kIHN0YW5kYXJkaXplIHRoZSB2YXJpYWJsZXMgdG8gbWFrZSB0aGVpciBzY2FsZSBjb21wYXJhYmxlLgoKIyMgUiBjb2RlCmBgYHtyfQpsaWJyYXJ5KE1BU1MpCgojIEZpdCB0aGUgbW9kZWwKbW9kZWwubGRhIDwtIGxkYShTcGVjaWVzIH4uLCBkYXRhID0gdHJhaW4udHJhbnNmb3JtZWQpCiMgTWFrZSBwcmVkaWN0aW9ucwpwcmVkaWN0aW9ucyA8LSBtb2RlbC5sZGEgJT4lIHByZWRpY3QodGVzdC50cmFuc2Zvcm1lZCkKCiMgTW9kZWwgYWNjdXJhY3kKbWVhbihwcmVkaWN0aW9ucyRjbGFzcyA9PSB0ZXN0LnRyYW5zZm9ybWVkJFNwZWNpZXMpCmBgYAoKYGBge3J9Cm1vZGVsLmxkYQpgYGAKCmBgYHtyfQpzdW1tYXJ5KG1vZGVsLmxkYSkKYGBgCgoKTERBIGRldGVybWluZXMgZ3JvdXAgbWVhbnMgYW5kIGNvbXB1dGVzLCBmb3IgZWFjaCBpbmRpdmlkdWFsLCB0aGUgcHJvYmFiaWxpdHkgb2YgYmVsb25naW5nIHRvIHRoZSBkaWZmZXJlbnQgZ3JvdXBzLiBUaGUgaW5kaXZpZHVhbCBpcyB0aGVuIGFmZmVjdGVkIHRvIHRoZSBncm91cCB3aXRoIHRoZSBoaWdoZXN0IHByb2JhYmlsaXR5IHNjb3JlLgoKVGhlICoqbGRhKCkqKiBvdXRwdXRzIGNvbnRhaW4gdGhlIGZvbGxvd2luZyBlbGVtZW50czoKKiAqKnByaW9yIHByb2JhYmlsaXRpZXMgb2YgZ3JvdXBzKio6IHRoZSBwcm9wb3J0aW9uIG9mIHRyYWluaW5nIG9ic2VydmF0aW9uIGluIGVhY2ggZ3JvdXAuIEZvciBleGFtcGxlLCB0aGVyZSBhcmUgMzMlIG9mIHRoZSB0cmFpbmluZyBvYnNlcnZhdGlvbnMgaW4gdGhlIHNldG9zYSBncm91cAoqICoqR3JvdXAgbWVhbnMqKjogZ3JvdXAgY2VudGVyIG9mIGdyYXZpdHkuIFNob3dzIHRoZSBtZWFuIG9mIGVhY2ggdmFyaWFibGUgaW4gZWFjaCBncm91cC4KKiAqKkNvZWZmaWNpZW50cyBvZiBsaW5lYXIgZGlzY3JpbWluYW50cyoqOiBTaG93cyB0aGUgbGluZWFyIGNvbWJpbmF0aW9uIG9mIHByZWRpY3RvciB2YXJpYWJsZXMgdGhhdCBhcmUgdXNlZCB0byBmb3JtIHRoZSBMREEgZGVjaXNpb25ydWxlLiBGb3IgZXhhbXBsZQoKTEQxID0gMC4wMVwqU2VwYWwuTGVuZ3RoICsgMC42NFwqU2VwYWwuV2lkdGggLSA0LjA4XCpQZXRhbC5MZW5ndGggPSAyLjNcKlBldGFsLldpZHRoCkxEMiA9IDAuMDNcKlNlcGFsLkxlbmd0aCArIDAuODlcKlNlcGFsLldpZHRoIC0gMi4yXCpQZXRhbC5MZW5ndGggLSAyLjZcKlBldGFsLldpZHRoCgpVc2luZyB0aGUgZnVuY3Rpb24gKipQbG90KCkqKiBwcm9kdWNlcyBwbG90cyBvZiB0aGUgbGluZWFyIGRpc2NyaW1pbmFudHMsIG9idGFpbmVkIGJ5IGNvbXB1dGluZyBMRDEgYW5kIExEMiBmb3IgZWFjaCBvZiB0aGUgdHJhaW5pbmcgb2JzZXJ2YXRpb25zCgpgYGB7cn0KcGxvdChtb2RlbC5sZGEpCmBgYAoKYGBge3J9CmxkYS5kYXRhIDwtIGNiaW5kKHRyYWluLnRyYW5zZm9ybWVkLCBwcmVkaWN0KG1vZGVsLmxkYSkkeCkKCmdncGxvdChsZGEuZGF0YSwgYWVzKExEMSwgTEQyKSkgKwogIGdlb21fcG9pbnQoYWVzKGNvbG9yID0gU3BlY2llcykpCmBgYAoKCj4gSXQgY2FuIGJlIHNlZW4gdGhhdCwgb3VyIG1vZGVsIGNvcnJlY3RseSBjbGFzc2lmaWVkIDEwMCUgb2Ygb2JzZXJ2YXRpb25zLCB3aGljaCBpcyBleGNlbGxlbnQuCgpOb3RlIHRoYXQsIGJ5IGRlZmF1bHQsIHRoZSBwcm9iYWJpbGl0eSBjdXRvZmYgdXNlZCB0byBkZWNpZGUgZ3JvdXAtbWVtYmVyc2hpcCBpcyAwLjUgCgpJbiBzb21lIHNpdHVhdGlvbiwgeW91IG1pZ2h0IHdhbnQgdG8gaW5jcmVhc2UgdGhlIHByZWNpc2lvbiBvZiB0aGUgbW9kZWwuIEluIHRoaXMgY2FzZSB5b3UgY2FuIGZpbmUtdHVuZSB0aGUgbW9kZWwgYnkgYWRqdXN0aW5nIHRoZSBwb3N0ZXJpb3IgcHJvYmFiaWxpdHkgY3V0b2ZmLiBGb3IgZXhhbXBsZSwgeW91IGNhbiBpbmNyZWFzZSBvciBsb3dlciB0aGUgY3V0b2ZmCgojIyMjIFZhcmlhYmxlIHNlbGVjdGlvbjoKCk5vdGUgdGhhdCwgaWYgdGhlIHByZWRpY3RvciB2YXJpYWJsZXMgYXJlIHN0YW5kYXJkaXplZCBiZWZvcmUgY29tcHV0aW5nIExEQSwgdGhlIGRpc2NyaW1pbmF0b3Igd2VpZ2h0cyBjYW4gYmUgdXNlZCBhcyBtZWFzdXJlcyBvZiB2YXJpYWJsZSBpbXBvcnRhbmNlIGZvciBmZWF0dXJlIHNlbGVjdGlvbgoKIyMjIFF1YXJkcmF0aWMgZGlzY3JpbWluYW50IGFuYWx5c2lzIC0gUURBCgpRREEgaXMgbGl0dGxlIGJpdCBtb3JlIGZsZXhpYmxlIHRoYW4gTERBLCBpbiB0aGUgc2Vuc2UgdGhhdCBpdCBkb2VzIG5vdCBhc3N1bWVzIHRoZSBlcXVhbGl0eSBvZiB2YXJpYW5jZS9jb3ZhcmlhbmNlLiBJbiBvdGhlciB3b3JkcywgZm9yIFFEQSB0aGUgY292YXJpYW5jZSBtYXRyaXggY2FuIGJlIGRpZmZlcmVudCBmb3IgZWFjaCBjbGFzcy4KCkxEQSB0ZW5kcyB0byBiZSBhIGJldHRlciB0aGFuIFFEQSB0aGVuIHlvdSBoYXZlIGEgc21hbGwgdHJhaW5pbmcgc2V0LgoKSW4gY29udHJhc3QsIFFEQSBpcyByZWNvbW1lbmRlZCBpZiB0aGUgdHJhaW5pbmcgc2V0IGlzIHZlcnkgbGFyZ2UsIHNvIHRoYXQgdGhlIHZhcmlhbmNlIG9mIHRoZSBjbGFzc2lmaWVyIGlzIG5vdCBhIG1ham9yIGlzc3VlLCBvciBpZiB0aGUgYXNzdW1wdGlvbiBvZiBhIGNvbW1vbiBjb3ZhcmlhbmNlIG1hdHJpeCBmb3IgdGhlIEsgY2xhc3NlcyBpZiBjbGVhcmx5IHVudGVuYWJsZQoKUURBIGNhbiBiZSBjb21wdXRlZCB1c2luZyB0aGUgUiBmdW5jdGlvbiBxZGEoKSBbTUFTUyBwYWNrYWdlXQoKYGBge3J9CmxpYnJhcnkoTUFTUykKI0ZpdCB0aGUgbW9kZWwKbW9kZWwucWRhIDwtIHFkYShTcGVjaWVzfi4sIGRhdGEgPSB0cmFpbi50cmFuc2Zvcm1lZCkKbW9kZWwucWRhCgojIE1ha2UgcHJlZGljdGlvbnMKcHJlZGljdGlvbnMucWRhIDwtIG1vZGVsLnFkYSAlPiUgcHJlZGljdCh0ZXN0LnRyYW5zZm9ybWVkKQoKIyBNb2RlbCBhY2N1cmFjeQptZWFuKHByZWRpY3Rpb25zLnFkYSRjbGFzcyA9PSB0ZXN0LnRyYW5zZm9ybWVkJFNwZWNpZXMpCmBgYAoKCiMjIE1peHR1cmUgZGlzY3JpbWluYW50IGFuYWx5c2lzCgpUaGUgTERBIGNsYXNzaWZpZXIgYXNzdW1lcyB0aGF0IGVhY2ggY2xhc3MgY29tZXMgZnJvbSBhIHNpbmdsZSBub3JtYWwgKG9yIEdhdXNzaWFuICkgZGlzdHJpYnV0aW9uLgoKVGhpcyBpcyB0b28gcmVzdHJpY3RpdmUuCgpGb3IgTURBLCB0aGVyIGFyZSBjbGFzc2VzLCBhbmQgZWFjaCBjbGFzcyBpcyBhc3N1bWVkIHRvIGJlIGEgR2F1c3NpYW4gbWl4dHVyZSBvZiBzdWJjbGFzc2VzLCB3aGVyZSBlYWNoIGRhdGEgcG9pbnQgaGFzIGEgcHJvYmFiaWxpdHkgb2YgYmVsb25naW5nIHRvIGVhY2ggY2xhc3MuIEVxdWFsaXR5IG9mIGNvdmFyaWFuY2UgbWF0cml4LCBhbW9uZyBjbGFzc2VzLCBpcyBzdGlsbCBhc3N1bWVkCgpgYGB7cn0KbGlicmFyeShtZGEpCm1vZGVsLm1kYSA8LSBtZGEoU3BlY2llc34uLCBkYXRhID0gdHJhaW4udHJhbnNmb3JtZWQpCm1vZGVsLm1kYQoKIyBNYWtlIHByZWRpY3Rpb25zCgpwcmVkaWN0ZWQuY2xhc3Nlcy5tZGEgPC0gbW9kZWwubWRhICU+JSBwcmVkaWN0KHRlc3QudHJhbnNmb3JtZWQpCgojIE1vZGVsIGFjY3VyYWN5Cm1lYW4ocHJlZGljdGVkLmNsYXNzZXMubWRhID09IHRlc3QudHJhbnNmb3JtZWQkU3BlY2llcykKYGBgCgoKIyMjIEZsZXhpYmxlIGRpc2NyaW1pbmFudCBhbmFseXNpcyAtIEZEQSAKCkZEQSBpcyBhIGZsZXhpYmxlIGV4dGVuc2lvbiBvZiBMREEgdGhhdCB1c2VzIG5vbi1saW5lYXIgY29tYmluYXRpb25zIG9mIHByZWRpY3RvcnMgc3VjaCBhcyBzcGxpbmVzLiBGREEgaXMgdXNlZnVsIHRvIG1vZGVsIG11bHRpdmFyaWF0ZSBub24tbm9ybWFsaXR5IG9yIG5vbi1saW5lYXIgcmVsYXRpb25zaGlwcyBhbW9uZyB2YXJpYWJsZXMgd2l0aGluIGVhY2ggZ3JvdXAsIGFsb3dpbmcgZm9yIGEgbW9yZSBhY2N1cmF0ZSBjbGFzc2lmaWNhdGlvbgoKYGBge3J9CmxpYnJhcnkobWRhKQojIEZpdCB0aGUgbW9kZWwKbW9kZWwuZmRhIDwtIGZkYShTcGVjaWVzfi4sIGRhdGEgPSB0cmFpbi50cmFuc2Zvcm1lZCkKbW9kZWwuZmRhCgpwcmVkaWN0ZWQuY2xhc3Nlcy5mZGEgPC0gbW9kZWwuZmRhICU+JSBwcmVkaWN0KHRlc3QudHJhbnNmb3JtZWQpCiNNb2RlbCBhY2N1cmFjeQptZWFuKHByZWRpY3RlZC5jbGFzc2VzLmZkYSA9PSB0ZXN0LnRyYW5zZm9ybWVkJFNwZWNpZXMpCmBgYAoKCiMjIyBSZWd1bGFyaXplZCBkaXNjcmltaW5hbnQgYW5hbHlzaXMKClJEQSBidWlsZHMgYSBjbGFzc2lmaWNhdGlvbiBydWxlIGJ5IHJlZ3VsYXJpemluZyB0aGUgZ3JvdXAgY292YXJpYW5jZSBtYXRyaWNlcyBhbGxvd2luZyBhIG1vcmUgcm9idXN0IG1vZGVsIGFnYWluc3QgbXVsdGljb2xsaW5lYXJpdHkgaW4gdGhlIGRhdGEuIFRoaXMgbWlnaHQgYmUgdmVyeSB1c2VmdWwgZm9yIGEgbGFyZ2UgbXVsdGl2YXJpYXRlIGRhdGEgc2V0IGNvbnRhaW5pbmcgaGlnaGx5IGNvcnJlbGF0ZWQgcHJlZGljdG9ycy4KClJlZ3VsYXJpemVkIGRpc2NyaW1pbmFudCBhbmFseXNpcyBpcyBhIGtpbmQgb2YgYSB0cmFkZS1vZmYgYmV0d2VlbiBMREEgYW5kIFFEQS4gUmVjYWxsIHRoYXQsIGluIExEQSB3ZSBhc3N1bWUgZXF1YWxpdHkgb2YgY292YXJpYW5jZSBtYXRyaXggZm9yIGFsbCBvZiB0aGUgY2xhc3Nlcy4gUURBIGFzc3VtZXMgZGlmZmVyZW50IGNvdmFyaWFuY2UgbWF0cmljZXMgZm9yIGFsbCB0aGUgY2xhc3Nlcy4gUURhIGFzc3VtZXMgZGlmZmVyZW50IGNvdmFyaWFuY2UgbWF0cmljZXMgZm9yIGFsbCB0aGUgY2xhc3Nlcy4gUmVndWxhcml6ZWQgZGlzY3JpbWluYW50IGFuYWx5c2lzIGlzIGFuIGludGVybWVkaWF0ZSBiZXR3ZWVuIExEQSBhbmQgUURBLgoKUkRBIHNocmlua3MgdGhlIHNlcGFydGUgY292YXJpYW5jZXMgb2YgUURBIHRvd2FyZCBhIGNvbW1vbiBjb3ZhcmlhbmNlIGFzIGluIExEQS4gVGhpcyBpbXByb3ZlcyB0aGUgZXN0aW1hdGUgb2YgdGhlIGNvdmFyaWFuY2UgbWF0cmljZXMgaW4gc2l0dWF0aW9ucyB3aGVyZSB0aGUgbnVtYmVyIG9mIHByZWRpY3RvcnMgaXMgbGFyZ2VyIHRoYW50IHRoZSBudW1iZXIgb2Ygc2FtcGxlcyBpbiB0aGUgdHJhaW5pbmcgZGF0YSwgcG90ZW50aWFsbHkgbGVhZGluZyB0byBhbiBpbXByb3ZlbWVudCBvZiB0aGUgbW9kZWwgYWNjdXJhY3kuCgpgYGB7cn0KbGlicmFyeShrbGFSKQojIEZpdCB0aGUgbW9kZWwKbW9kZWwucmRhIDwtIHJkYShTcGVjaWVzfi4sIGRhdGEgPSB0cmFpbi50cmFuc2Zvcm1lZCkKCiMgIE1ha2UgcHJlZGljdGlvbnMKcHJlZGljdGlvbnMucmRhIDwtIG1vZGVsLnJkYSAlPiUgcHJlZGljdCh0ZXN0LnRyYW5zZm9ybWVkKQoKIyBNb2RlbCBhY2N1cmFjeQoKbWVhbihwcmVkaWN0aW9ucy5yZGEkY2xhc3MgPT0gdGVzdC50cmFuc2Zvcm1lZCRTcGVjaWVzKQpgYGAKCgojIyBOYWl2ZSBCYXllcyBDbGFzc2lmaWVyCgpUaGUgKipOYWl2ZSBCYXllcyBDbGFzc2lmaWVyKiogaXMgYSBzaW1wbGUgYW5kIHBvd2VyZnVsIG1ldGhvZCB0aGF0IGNhbiBiZSB1c2VkIGZvciBiaW5hcnkgYW5kIG11bHRpY2xhc3MgY2xhc3NpZmljYXRpb24gcHJvYmxlbXMuCgpOYWl2ZSBCYXllcyBjbGFzc2lmaWVyIHByZWRpY3RzIHRoZSBjbGFzcyBtZW1iZXJzaGlwIHByb2JhYmlsaXR5IG9mIG9ic2VydmF0aW9ucyB1c2luZyBCYXllcyB0aGVvcmVtLCB3aGljaCBpcyBiYXNlZCBvbiBjb25kaXRpb25hbCBwcm9iYWJpbGl0eSwgdGhhdCBpcyB0aGUgcHJvYmFiaWxpdHkgb2Ygc29tZXRoaW5nIHRvIGhhcHBlbiwgZ2l2ZW4gdGhhdCBzb21ldGhpbmcgZWxzZSBoYXMgYWxyZWFkeSBvY2N1cmVkLgoKCldpbGwgdXNlIFBpbWFJbmRpYW5zRGlhYmV0ZXMyIGRhdGEgc2V0CgojIyMjIENvbXB1dGluZyBOYWl2ZSBCYXllcwoKYGBge3J9CmxpYnJhcnkoa2xhUikKIyBGaXQgdGhlIG1vZGVsCm1vZGVsLm5haXZlIDwtIE5haXZlQmF5ZXMoZGlhYmV0ZXMgfi4sIGRhdGEgPSB0cmFpbi5waW1hLmluZGlhbnMpCgptb2RlbC5uYWl2ZQoKYGBgCgpgYGB7cn0KIyBNYWtlIHByZWRpY3Rpb25zCnByZWRpY3Rpb24ubmFpdmUgPC0gbW9kZWwubmFpdmUgJT4lIHByZWRpY3QodGVzdC5waW1hLmluZGlhbnMpCgojIE1vZGVsIGFjY3VyYWN5Cm1lYW4ocHJlZGljdGlvbi5uYWl2ZSRjbGFzcyA9PSB0ZXN0LnBpbWEuaW5kaWFucyRkaWFiZXRlcykKYGBgCgojIyMjIFVzaW5nIGNhcmV0IFIgcGFja2FnZQoKVGhlIGNhZXQgUiBwYWNrYWdlIGNhbiBhdXRvbWF0aWNhbGx5IHRyYWluIHRoZSBtb2RlbCBhbmQgYXNzZXNzIHRoZSBtb2RsZSBhY2N1cmFjeSB1c2luZyBrLWZvbGQgY3Jvc3MtdmFsaWRhdGlvbgoKYGBge3J9CmxpYnJhcnkoa2xhUikKCnNldC5zZWVkKDEyMykKCm1vZGVsLm5haXZlMiA8LSB0cmFpbihkaWFiZXRlc34uLCBkYXRhID0gdHJhaW4ucGltYS5pbmRpYW5zLCBtZXRob2QgPSAibmIiLCB0ckNvbnRyb2wgPSB0cmFpbkNvbnRyb2woImN2IiwgbnVtYmVyID0gMTApKQoKIyBtYWtlIHByZWRpY3Rpb25zCnByZWRpY3RlZC5jbGFzc2VzLm5haXZlMiA8LSBtb2RlbC5uYWl2ZTIgJT4lIHByZWRpY3QodGVzdC5waW1hLmluZGlhbnMpCgojIE1vZGVsIG4gYWNjdXJhY3kKCm1lYW4ocHJlZGljdGVkLmNsYXNzZXMubmFpdmUyID09IHRlc3QucGltYS5pbmRpYW5zJGRpYWJldGVzKQpgYGAKCgojIyMgU3VwcG9ydCBWZWN0b3IgTWFjaGluZQoKKipTdXBwb3J0IFZlY3RvciBNYWNoaW5lIChvciBTVk0pKiogaXMgYSBtYWNoaW5lIGxlYXJuaW5nIHRlY2huaXF1ZSB1c2VkIGZvciBjbGFzc2lmaWNhdGlvbiB0YXNrcy4gQnJpZWZseSwgU1ZNIHdvcmtzIGJ5IGlkZW50aWZ5aW5nIHRoZSBvcHRpbWFsIGRlY2lzaW9uIGJvdW5kYXJ5IHRoYXQgc2VwYXJhdGVzIGRhdGEgcG9pbnRzIGZyb20gZGlmZmVyZW50IGdyb3VwcyAob3IgY2xhc3NlcyksIGFuZCB0aGVuIHByZWRpY3RzIHRoZSBjbGFzcyBvZiBuZXcgb2JzZXJ2YXRpb25zIGJhc2VkIG9uIHRoZSBzZXBhcmF0aW9uIGJvdW5kYXJ5LgoKRGVwZW5kaW5nIG9uIHRoZSBzaXR1YXRpb25zLCB0aGUgZGlmZmVyZW50IGdyb3VwcyBtaWdodCBiZSBzZXBhcmFibGUgYnkgYSBsaW5lYXIgc3RyYWlnaHQgbGluZSBvciBieSBhIG5vbi1saW5lYXIgYm91bmRhcnkgbGluZS4KClN1cHBvcnQgdmVjdG9yIG1hY2hpbmUgbWV0aG9kcyBjYW4gaGFuZGxlIGJvdGggbGluZWFyIGFuZCBub24tbGluZWFyIGNsYXNzIGJvdW5kYXJpZXMuIEl0IGNhbiBiZSB1c2VkIGZvciBib3RoIHR3by1jbGFzc3MgYW5kIG11bHRpLWNsYXNzIGNsYXNzaWZpY2F0aW9uIHByb2JsZW1zLgoKSW4gcmVhbCBsaWZlIGRhdGEsIHRoZSBzZXBhcmF0aW9uIGJvdW5kYXJ5IGlzIGdlbmVyYWxseSBub25saW5lYXIuIFRlY2huaWNhbGx5LCB0aGUgU1ZNIGFsZ29yaXRobSBwZXJmb3JtIGEgbm9uLWxpbmVhciBjbGFzc2lmaWNhdGlvbiB1c2luZyB3aGF0IGlzIGNhbGxlZCB0aGUga2VybmVsIHRyaWNrLiBUaGUgbW9zdCBjb21tb25seSB1c2VkIGtlcm5lbCB0cmFuc2Zvcm1hdGlvbnMgYXIgZV9wb2x5bm9taWFsIGtlcm5lbF8gYW5kIF9yYWRpYWwga2VybmVsXwoKTm90ZSB0aGF0LCB0aGVyZSBpcyBhbHNvIGFuIGV4dGVuc2lvbiBvZiB0aGUgU1ZNIGZvciByZWdyZXNzaW9uLCBjYWxsZWQgc3VwcG9ydCB2ZWN0b3IgcmVncmVzc2lvbi4KCiMjIyMgTG9hZGluZyByZXF1aXJlZCBSIHBhY2thZ2VzCgpgYGB7cn0KbGlicmFyeSh0aWR5dmVyc2UpCmxpYnJhcnkoY2FyZXQpCmBgYAoKCiMjIyMgRGF0YXNldAoKRGF0YSBzZXQ6IFBpbWFJbmRpYW5zRGlhYmV0ZXMyIFtpbiBtbGJlbmNoIHBhY2thZ2VdCgpgYGB7cn0KIyBMb2FkIHRoZSBkYXRhCmRhdGEoIlBpbWFJbmRpYW5zRGlhYmV0ZXMyIiwgcGFja2FnZSA9ICJtbGJlbmNoIikKcGltYS5kYXRhLmNsZWFuZWQgPC0gbmEub21pdChQaW1hSW5kaWFuc0RpYWJldGVzMikKCiMgU3BsaXQgdGhlIGRhdGEgaW50byB0cmFpbmluZyBhbmQgdGVzdCBzZXQKc2V0LnNlZWQoMTIzKQoKdC5zYW1wbGUgPC0gcGltYS5kYXRhLmNsZWFuZWQkZGlhYmV0ZXMgJT4lIGNyZWF0ZURhdGFQYXJ0aXRpb24ocCA9IDAuOCwgbGlzdCA9IEZBTFNFKQpgYGAKCiMjIyMgU1ZNIGxpbmVhciBjbGFzc2lmaWVyCgpJbiB0aGUgZm9sbG93aW5nIGV4YW1wbGUgdmFyaWFibGVzIGFyZSBub3JtYWxpemVkIHRvIG1ha2UgdGhlaXIgc2NhbGUgY29tcGFyYWJsZS4gVGhpcyBpcyBhdXRvbWF0aWNhbGx5IGRvbmUgYmVmb3JlIGJ1aWxkaW5nIHRoZSBTVk0gY2xhc3NpZmllciBieSBzZXR0aW5nIHRoZSBvcHRpb24gcHJlUHJvY2VzcyA9IGMoImNlbnRlciIsICJzY2FsZSIpLgoKYGBge3J9CiMgRml0IHRoZSBtb2RlbCBvbiB0aGUgdHJhaW5pbmcgc2V0CnNldC5zZWVkKDEyMykKCm1vZGVsLnN2bSA8LSB0cmFpbigKICBkaWFiZXRlc34uLCBkYXRhID0gdHJhaW4ucGltYS5pbmRpYW5zLCBtZXRob2QgPSAic3ZtTGluZWFyIiwKICB0ckNvbnRybyA9IHRyYWluQ29udHJvbCgiY3YiLCBudW1iZXIgPSAxMCksCiAgcHJlUHJvY2VzcyA9IGMoImNlbnRlciIsICJzY2FsZSIpCikKCiMgTWFrZSBwcmVkaWN0aW9ucyBvbiB0aGUgdGVzdCBkYXRhCnByZWRpY3RlZC5jbGFzc2VzLnN2bSA8LSBtb2RlbC5zdm0gJT4lIHByZWRpY3QodGVzdC5waW1hLmluZGlhbnMpCmhlYWQocHJlZGljdGVkLmNsYXNzZXMuc3ZtKQpgYGAKCmBgYHtyfQojIENvbXB1dGVkIG1vZGVsIGFjY3VyYWN5IHJhdGUKbWVhbihwcmVkaWN0ZWQuY2xhc3Nlcy5zdm0gPT0gdGVzdC5waW1hLmluZGlhbnMkZGlhYmV0ZXMpCmBgYAoKTm90ZSB0aGF0LCB0aGVyZSBpcyBhIHR1bmluZyBwYXJhbWV0ZXIgQywgYWxzbyBrbm93biBhcyBfQ29zdF8sIHRoYXQgZGV0ZXJtaW5lcyB0aGUgcG9zc2libGUgbWlzY2xhc3NpZmljYXRpb25zLiBJdCBlc3NlbnRpYWxseSBpbXBvc2VzIGEgcGVuYWx0eSB0byB0aGUgbW9kZWwgZm9yIG1ha2luZyBhbiBlcnJvci4gVGhlIGhpZ2hlciB0aGUgdmFsdWUgb2YgQywgdGhlIGxlc3MgbGlrZWx5IGl0IGlzIHRoYXQgdGhlIFNWTSBhbGdvcml0aG0gd2lsbCBtaXNjbGFzc2lmeSBhIHBvaW50LgoKQnkgZGVmYXVsdCAqKmNhcmV0KiogYnVpbGRzIHRoZSBTVk0gbGluZWFyIGNsYXNzaWZpZXIgdXNpbiBnIEM9IDEuIFlvdSBjYW4gY2hlY2sgdGhpcyBieSB0eXBpbmcgbW9kZWwgaW4gUiBjb25zb2xlCgpgYGB7cn0KbW9kZWwuc3ZtCmBgYAoKVGhlIGZvbGxvd2luZyBSIGNvZGUgY29tcHV0ZSBTVk0gZm9yIGEgZ3JpZCB2YWx1ZXMgb2YgQyBhbmQgY2hvb3NlIGF1dG9tYXRpY2FsbHkgdGhlIGZpbmFsIG1vZGVsIGZvciBwcmVkaWN0aW9uczoKCmBgYHtyfQojIEZpdCB0aGUgbW9kZWwgb24gdGhlIHRyYWluaW5nIHNldApzZXQuc2VlZCgxMjMpCgptb2RlbC5zdm0yIDwtIHRyYWluKAogIGRpYWJldGVzIH4uLCBkYXRhID0gdHJhaW4ucGltYS5pbmRpYW5zLCBtZXRob2QgPSAic3ZtTGluZWFyIiwKICB0ckNvbnRyb2wgPSB0cmFpbkNvbnRyb2woImN2IiwgbnVtYmVyID0gMTApLAogIHR1bmVHcmlkID0gZXhwYW5kLmdyaWQoQyA9IHNlcSgwLjEsIDIsIGxlbmd0aCA9IDIwKSksCiAgcHJlUHJvY2VzcyA9IGMoImNlbnRlciIsICJzY2FsZSIpCikKCiMgUGxvdCBtb2RlbCBhY2N1cmFjeSB2cyBkaWZmZXJlbnQgdmFsdWVzIG9mIENvc3QKcGxvdChtb2RlbC5zdm0yKQpgYGAKCgpgYGB7cn0KIyBQcmludCB0aGUgYmVzdCB0dW5pbmcgcGFyYW1ldGVyIEMgdGhhdCBtYXhpbWl6ZXMgbW9kZWwgYWNjdXJhY3kKbW9kZWwuc3ZtMiRiZXN0VHVuZQpgYGAKCgpMZXQncyB1c2UgdGhlIGJlc3QgbW9kZWwgCgpgYGB7cn0KIyBNYWtlIHByZWRpY3Rpb25zIG9uIHRoZSB0ZXN0IGRhdGEgCnByZWRpY3RlZC5jbGFzc2Vzcy5zdm0yIDwtIG1vZGVsLnN2bTIgJT4lIHByZWRpY3QodGVzdC5waW1hLmluZGlhbnMpCgojIENvbXB1dGUgbW9kZWwgYWNjdXJhY3kgcmF0ZQptZWFuKHByZWRpY3RlZC5jbGFzc2Vzcy5zdm0yID09IHRlc3QucGltYS5pbmRpYW5zJGRpYWJldGVzKQpgYGAKCgojIyMgU1ZNIGNsYXNzaWZpZXIgdXNpbmcgTm9uLUxpbmVhciBLZXJuZWwKClRvIGJ1aWxkIGEgbm9uLWxpbmVhciBTVk0gY2xhc3NpZmllciwgd2UgY2FuIHVzZSBlaXRoZXIgcG9seW5vbWlhbCBrZXJuZWwgb3IgcmFkaWFsIGtlcm5lbCBmdW5jdGlvbi4gCgoqICoqQ29tcHV0aW5nIFNWTSB1c2luZyByYWRpYWwgYmFzaXMga2VybmVsKioKCmBgYHtyfQojIEZpdCB0aGUgbW9kZWwgb24gdGhlIHRyYWluaW5nIHNldApzZXQuc2VlZCgxMjMpCm1vZGVsLnN2bS5yYWRpYWwgPC0gdHJhaW4oCiAgZGlhYmV0ZXN+LiwgZGF0YSA9IHRyYWluLnBpbWEuaW5kaWFucywgbWV0aG9kID0gInN2bVJhZGlhbCIsCiAgdHJDb250cm9sID0gdHJhaW5Db250cm9sKCJjdiIsIG51bWJlciA9IDEwKSwKICBwcmVQcm9jZXNzID0gYygiY2VudGVyIiwgInNjYWxlIiksCiAgdHVuZUxlbmd0aCA9IDEwCikKCiMgUHJpbnQgdGhlIGJlc3QgdHVuaW5nIHBhcmFtZXRlciBzaWdtYSBhbmQgQyB0aGF0IG1heGltaXplcyBtb2RlbCBhY2N1cmFjeQoKbW9kZWwuc3ZtLnJhZGlhbCRiZXN0VHVuZQpgYGAKCgpgYGB7cn0KIyBNYWtlIHByZWRpY3Rpb25zIG9uIHRoZSB0ZXN0IGRhdGEKcHJlZGljdGVkLmNsYXNzZXNzLnN2bS5yYWRpYWwgPC0gbW9kZWwuc3ZtLnJhZGlhbCAlPiUgcHJlZGljdCh0ZXN0LnBpbWEuaW5kaWFucykKIyBDb21wdXRlIG1vZGVsIEFjY3VyYWN5IHJhdGUKbWVhbihwcmVkaWN0ZWQuY2xhc3Nlc3Muc3ZtLnJhZGlhbCA9PSB0ZXN0LnBpbWEuaW5kaWFucyRkaWFiZXRlcykKYGBgCgoqICoqQ29tcHV0aW5nIFNWTSB1c2luZyBwb2x5bm9taWFsIGJhc2lzIGtlcm5lbCoqOgoKYGBge3J9CiMgRml0IHRoZSBtb2RlbCBvbiB0aGUgdHJhaW5pbmcgc2V0CnNldC5zZWVkKDEyMykKbW9kZWwuc3ZtLnBvbHlub21pYWwgPC0gdHJhaW4oCiAgZGlhYmV0ZXN+LiwgZGF0YSA9IHRyYWluLnBpbWEuaW5kaWFucywgbWV0aG9kID0gInN2bVBvbHkiLAogIHRyQ29udHJvbCA9IHRyYWluQ29udHJvbCgiY3YiLCBudW1iZXIgPSAxMCksCiAgcHJlUHJvY2VzcyA9IGMoImNlbnRlciIsICJzY2FsZSIpLAogIHR1bmVMZW5ndGggPSA0CikKCiMgUHJpbnQgdGhlIGJlc3QgdHVuaW5nIHBhcmFtZXRlciBzaWdtYSBhbmQgQyB0aGF0IG1heGltaXplcyBtb2RlbCBhY2N1cmFjeQoKbW9kZWwuc3ZtLnBvbHlub21pYWwkYmVzdFR1bmUKYGBgCgoKCgpgYGB7cn0KIyBNYWtlIHByZWRpY3Rpb25zIG9uIHRoZSB0ZXN0IGRhdGEKcHJlZGljdGVkLmNsYXNzZXNzLnN2bS5wb2x5bm9taWFsIDwtIG1vZGVsLnN2bS5wb2x5bm9taWFsICU+JSBwcmVkaWN0KHRlc3QucGltYS5pbmRpYW5zKQojIENvbXB1dGUgbW9kZWwgQWNjdXJhY3kgcmF0ZQptZWFuKHByZWRpY3RlZC5jbGFzc2Vzcy5zdm0ucG9seW5vbWlhbCA9PSB0ZXN0LnBpbWEuaW5kaWFucyRkaWFiZXRlcykKYGBgCgoKPiBJbiBvdXIgZXhhbXBsZXMsIGl0IGNhbiBiZSBzZWVuIHRoYXQgdGhlIFNWTSBjbGFzc2lmaWVyIHVzaW5nIG5vbi1saW5lYXIga2VybmVsIGdpdmVzIGEgYmV0dGVyIHJlc3VsdCBjb21wYXJlZCB0byB0aGUgbGluZWFyIG1vZGVsLgoKCgojIyBDbGFzc2lmaWNhdGlvbiBNb2RlbCBFdmFsdWF0aW9uCgpBZnRlciBidWlsZGluZyBhIHByZWRpY3RpdmUgY2xhc3NpZmljYXRpb24gbW9kZWwsIHlvdSBuZWVkIHRvICoqZXZhbHVhdGUgdGhlIHBlcmZvcm1hbmNlIG9mIHRoZSBtb2RlbCoqLCB0aGF0IGlzIGhvdyBnb29kIHRoZSBtb2RlbCBpcyBpbiBwcmVkaWN0aW5nIHRoZSBvdXRjb21lIG9mIG5ldyBvYnNlcnZhdGlvbnMgdGVzdCBkYXRhIHRoYXQgaGF2ZSBiZWVuIG5vdCB1c2VkIHRvIHRyYWluIHRoZSBtb2RlbC4KClRoZSBjb21tb24gdXNlIG1ldHJpY3MgYW5kIG1ldGhvZHMgZm9yIGFzc2Vzc2luZyB0aGUgcGVyZm9ybWFuY2Ugb2YgcHJlZGljdGl2ZSBjbGFzc2lmaWNhdGlvbiBtb2RlbHM6CgoqICoqQXZlcmFnZSBjbGFzc2lmaWNhdGlvbiBhY2N1cmFjeSoqIC0gcmVwcmVzZW50aW5nIHRoZSBwcm9wb3J0aW9uIG9mIGNvcnJlY3RseSBjbGFzc2lmaWVkIG9ic2VydmF0aW9ucy4KCiogKipDb25mdXNpb24gbWF0cml4KiogLSB3aGljaCBpcyAyeDIgdGFibGUgc2hvd2luZyBmb3VyIHBhcmFtZXRlcnMsIGluY2x1ZGluZyB0aGUgbnVtYmVyIG9mIHRydWUgcG9zaXRpdmVzLCB0cnVlIG5lZ2F0aXZlcywgZmFsc2UgbmVnYXRpdmVzIGFuZCBmYWxzZSBwb3NpdGl2ZXMuCgoqICoqUHJlY2lzaW9uLCBSZWNhbGwgYW5kIFNwZWNpZmljaXR5KiogLSB3aGljaCBhcmUgdGhyZWUgbWFqb3IgcGVyZm9ybWFuY2UgbWV0cmljcyBkZXNjcmliaW5nIGEgcHJlZGljdGl2ZSBjbGFzc2lmaWNhdGlvbiBtb2RlbAoKKiAqKlJPQyBjdXJ2ZSoqIC0gd2hpY2ggaXMgYSBncmFwaGljYWwgc3VtbWFyeSBvZiB0aGUgb3ZlcmFsbCBwZXJmb3JtYW5jZSBvZiB0aGUgbW9kZWwsIHNob3dpbmcgdGhlIHByb3BvcnRpb24gb2YgdHJ1ZSBwb3NpdGl2ZXMgYW5kIGZhbHNlIHBvc2l0aXZlcyBhdCB0YWxsIHBvc3NpYmxlIHZhbHVlcyBvZiBwcm9iYWJpbGl0eSBjdXRvZmYuIFRoZSAqKkFyZWEgVW5kZXIgdGhlIEN1cnZlKEFVQykqKiBzdW1tYXJpemVzIHRoZSBvdmVyYWxsIHBlcmZvcm1hbmNlIG9mIHRoZSBjbGFzc2lmaWVyLgoKIyMjIyBSZXF1aXJlZCBSIHBhY2thZ2VzCgoqIHRpZHl2ZXJzZSBmb3IgZWFzeSBkYXRhIG1hbmlwdWxhdGlvbiBhbmQgdmlzdWFsaXphdGlvbgoqIGNhcmV0IGZvciBlYXN5IG1hY2hpbmUgbGVhcm5pbmcgd29ya2Zsb3cKCmBgYHtyfQpsaWJyYXJ5KHRpZHl2ZXJzZSkKbGlicmFyeShjYXJldCkKYGBgCgojIyMjIEJ1aWxkaW5nIGEgY2xhc3NpZmljYXRpb24gbW9kZWwKClRvIGtlZXAgdGhpbmdzIHNpbXBsZSB3ZSdsbCBwZXJmb3JtIGEgYmluYXJ5IGNsYXNzaWZpY2F0aW9uLCB3aGVyZSB0aGUgb3V0Y29tZSB2YXJpYWJsZSBjYW4gaGF2ZSBvbmx5IHR3byBwb3NzaWJsZSB2YWx1ZXM6IG5lZ2F0aXZlIHZzIHBvc2l0aXZlLgoKV2UnbGwgY29tcHV0ZSBhbiBleGFtcGxlIG9mIGxpbmVhciBkaXNjcmltaW5hbnQgYW5hbHlzaXMgbW9kZWwgdXNpbmcgdGhlICoqUGltYUluZGlhbnNEaWFiZXRlczIqKiBbbWxiZW5jaCBwYWNrYWdlXQoKMS4gU3BsaXQgdGhlIGRhdGEgaW50byB0cmFpbmluZyAoODAlLCB1c2VkIHRvIGJ1aWxkIHRoZSBtb2RlbCkgYW5kIHRlc3Qgc2V0ICgyMCUsIHVzZWQgdG8gZXZhbHVhdGUgdGhlIG1vZGVsIHBlcmZvcm1hbmNlKToKCmBgYHtyfQpkYXRhKCJQaW1hSW5kaWFuc0RpYWJldGVzMiIsIHBhY2thZ2UgPSAibWxiZW5jaCIpCnBpbWEuZGF0YS5jbGVhbmVkIDwtIG5hLm9taXQoUGltYUluZGlhbnNEaWFiZXRlczIpCiMgU3BsaXQgdGhlIGRhdGEgaW50byB0cmFpbmluZyBhbmQgdGVzdCBzZXQKc2V0LnNlZWQoMTIzKQoKdC5zYW1wbGUgPC0gcGltYS5kYXRhLmNsZWFuZWQkZGlhYmV0ZXMgJT4lIGNyZWF0ZURhdGFQYXJ0aXRpb24ocCA9IDAuOCwgbGlzdCA9IEZBTFNFKQoKdHJhaW4ucGltYTIgPC0gcGltYS5kYXRhLmNsZWFuZWRbdC5zYW1wbGUsIF0KdGVzdC5waW1hMiA8LSBwaW1hLmRhdGEuY2xlYW5lZFstdC5zYW1wbGUsIF0KYGBgCgoyLiBGaXQgdGhlIExEQSBtb2RlbCBvbiB0aGUgdHJhaW5pbmcgc2V0IGFuZCBtYWtlIHByZWRpdGlvbnMgb24gdGhlIHRlc3QgZGF0YToKCmBgYHtyfQpsaWJyYXJ5KE1BU1MpCiNGaXQgTERBCgpmaXQubGRhIDwtIGxkYShkaWFiZXRlcyB+LiwgZGF0YSA9IHRyYWluLnBpbWEyKQoKIyBNYWtlIHByZWRpY3Rpb25zIG9uIHRoZSB0ZXN0IGRhdGEKcHJlZGljdGlvbnMucGltYTIgPC0gcHJlZGljdChmaXQubGRhLCB0ZXN0LnBpbWEyKQpwcmVpY3Rpb25zLnByb2JhYmlsaXRpZXMyIDwtIHByZWRpY3Rpb25zLnBpbWEyJHBvc3RlcmlvclssMl0KcHJlZGljdGVkLmNsYXNzZXNzMiA8LSBwcmVkaWN0aW9ucy5waW1hMiRjbGFzcwpvYnNlcnZlZC5jbGFzc2VzMiA8LSB0ZXN0LnBpbWEyJGRpYWJldGVzCmBgYAoKIyMjIyBPdmVyYWxsIGNsYXNzaWZpY2F0aW9uIGFjY3VyYWN5ClRoZSBvdmVyYWxsICoqY2xhc3NpZmljYXRpb24gYWNjdXJhY3kqKiByYXRlIGNvcnJlc3BvbmRzIHRvIHRoZSBwcm9wb3J0aW9uIG9mIG9ic2VydmF0aW9ucyB0aGF0IGhhdmUgYmVlbiBjb3JyZWN0bHkgY2xhc3NpZmllZC4gRGV0ZXJtaW5pbmcgdGhlIHJhdyBjbGFzc2lmaWNhdGlvbiBhY2N1cmFjeSBpcyB0aGUgZmlyc3Qgc3RlcCBpbiBhc3Nlc3NpbmcgdGhlIHBlcmZvcm1hbmNlIG9mIGEgbW9kZWwuCgpgYGB7cn0KYWNjdXJhY3kgPC0gbWVhbihvYnNlcnZlZC5jbGFzc2VzMiA9PSBwcmVkaWN0ZWQuY2xhc3Nlc3MyKQphY2N1cmFjeQpgYGAKCmBgYHtyfQplcnJvciA8LSBtZWFuKG9ic2VydmVkLmNsYXNzZXMyICE9IHByZWRpY3RlZC5jbGFzc2VzczIpCmVycm9yCmBgYAoKPiBGcm9tIHRoZSBvdXRwdXQgYXZvdmUsIHRoZSBsaW5lYXIgZGlzY3JpbWFudCBhbmFseXNpcyBjb3JyZWN0bHkgcHJlZGljdGVkIHRoZSBpbmRpdmlkdWFsIG91dGNvbWUgaW4gODElIG9mIHRoZSBjYXNlcy4gVGhpcyBpcyBieSBmYXIgYmV0dGVyIHRoYW4gcmFuZG9tIGd1ZXNzaW5nLiBUaGUgbWlzY2xhc3NpZmljYXRpb24gZXJyb3IgcmF0ZSBjYW4gYmUgY2FsY3VsYXRlZCBhcyAxMDAtODEgPSAxOSUKCkluIG91ciBleGFtcGxlLCBhIGJpbmFyeSBjbGFzc2lmaWVyIGNhbiBtYWtlIHR3byB0eXBlcyBvZiBlcnJvcnM6CgoqIGl0IGNhbiBpbmNvcnJlY3RseSBhc3NpZ24gYW4gaW5kaXZpZHVhbCB3aG8gaXMgZGlhYmV0ZXMtcG9zaXRpdmUgdG8gdGhlIGRpYWJldGVzLW5lZ2F0aXZlIGNhdGVnb3J5CiogaXQgY2FuIGluY29ycmVjdGx5IGFzc2lnbiBhbiBpbmRpdmlkdWFsIHdobyBpcyBkaWFiZXRlcy1uZWdhdGl2ZSB0byB0aGUgZGlhYmV0ZXMtcG9zaXRpdmUgY2F0ZWdvcnkuCgpUaGUgcHJvcG9ydGlvbiBvZiB0aGVzZSB0d28gdHlwZXMgb2YgZXJyb3JzIGNhbiBiZSBkZXRlcm1pbmVkIGJ5IGNyZWF0aW5nIGEgKipjb25mdXRpb24gbWF0cnhpKiosIHdoaWNoIGNvbXBhcmUgdGhlIHByZWRpY3RlZCBjb3V0Y29tZSB2YWx1ZXMgYWdhaW5zIHRoZSBrbm93biBvdXRjb21lIHZhdWVzLgoKIyMjIENvbmZ1c2lvbiBtYXRyaXgKClRoZSBSIGZ1bmN0aW9uIHRhYmxlKCkgY2FuIGJlIHVzZWQgdG8gcHJvZHVjZSBhICoqY29uZnVzaW9uIG1hdHJpeCoqIGluIG9yZGVyIHRvIGRldGVybWluIGhvdyBtYW55IG9ic2VydmF0aW9ucyB3ZXJlIGNvcnJlY3RseSBvciBpbmNvcnJlY3RseSBjbGFzc2lmaWVkLiBJdCBjb21wYXJlcyB0aGUgb2JzZXJ2ZWQgYW5kIHRoZSBwcmVkaWN0ZWQgb3V0Y29tZSB2YWx1ZXMgYW5kIHNob3dzIHRoZSBudW1iZXIgb2YgY29ycmVjdCBhbmQgaW5jb3JyZWN0IHByZWRpY3Rpb25zIGNhdGVnb3JpemVkIGJ5IHR5cGUgb2Ygb3V0Y29tZS4KCmBgYHtyfQojIENvbmZ1c2lvbiBtYXRyaXgsIG51bWJlciBvZiBjYXNlcwp0YWJsZShvYnNlcnZlZC5jbGFzc2VzMiwgcHJlZGljdGVkLmNsYXNzZXNzMikKYGBgCgoKYGBge3J9CiMgQ29uZnVzaW9uIG1hdHJpeCwgcHJvcG9ydGlvbiBvZiBjYXNlcwp0YWJsZShvYnNlcnZlZC5jbGFzc2VzMiwgcHJlZGljdGVkLmNsYXNzZXNzMikgJT4lIHByb3AudGFibGUoKSAlPiUgcm91bmQoZGlnaXRzID0gMykKYGBgCgo+IFRoZSBkaWFnb25hbCBlbGVtZW50cyBvZiB0aGUgY29uZnVzaW9uIG1hdHJpeCBpbmRpY2F0ZSBjb3JyZWN0IHByZWRpY3Rpb25zLCB3aGlsZSB0aGUgb2ZmIGRpYWdvbmFscyByZXByZXNlbnQgaW5jb3JyZWN0IHByZWRpY3Rpb24uIFNvLCB0aGUgY29ycmVjdCBjbGFzc2lmaWNhdGlvbiByYXRlIGlzIHRoZSBzdW0gb2YgdGhlIG51bWJlciBvbiB0aGUgZGlhZ29uYWwgZGl2aWRlZCBieSB0aGUgc2FtcGxlIHNpemUgaW4gdGhlIHRlc3QgZGF0YS4gSW4gb3VyIGV4YW1wbGUsIHRoYXQgaXMgKDQ4KzE1KS83OCA9IDgxJQoKCiogKipUcnVlIHBvc2l0aXZlcyoqIChkKTogdGhlc2UgYXJlIGNhc2VzIGluIHdoaWNoIHdlIHByZWRpY3RlZCB0aGUgaW5kaXZpZHVhbHMgd291bGQgYmUgZGlhYmV0ZXMtcG9zaXRpdmUgYW5kIHRoZXkgd2VyZSAuCiogKipUcnVlIG5lZ2F0aXZlcyoqIChhKTogV2UgcHJlZGljdGVkIGRpYWJldGVzLW5lZ2F0aXZlLCBhbmQgdGhlIGluZGl2aWR1YWxzIHdlcmUgZGlhYmV0ZXMtbmVnYXRpdmUKKiAqKkZhbHNlIHBvc2l0aXZlcyoqIChiKTogV2UgcHJlZGljdGVkIGRpYWJldGVzLXBvc2l0aXZlLCBidXQgdGhlIGluZGl2aWR1YWxzIGRpZG4ndCBhY3R1YWxseSBoYWUgZGlhYmV0ZXMuIChBbHNvdyBrbm93biBhcyBhIF9UeXBlIEkgZXJyb3IuXykKKiAqKkZhbHNlIG5lZ2F0aXZlcyoqIChjKTogV2UgcHJlZGljdGVkIGRpYWJldGVzLW5lZ2F0aXZlLCBidXQgdGhleSBkaWQgaGF2ZSBkaWFiZXRlcy4gKEFsc28ga25vd24gYXMgYSBfVHlwZSBJSSBlcnJvci5fKQoKPlRlY2huaWNhbGx5IHRoZSByYXcgcHJlZGljdGlvbiBhY2N1cmFjeSBvZiB0aGUgbW9kZWwgaXMgZGVmaW5lZCBhcyAoVHJ1ZVBvc2l0aXZlcyArIFRydWVOZWdhdGl2ZXMpL1NhbXBsZVNpemUuCgoKIyMjIFByZWNpc2lvbiwgUmVjYWxsIGFuZCBTcGVjaWZpY2l0eQoKSW4gYWRkaXRpb24gdG8gdGhlIHJhdyBjbGFzc2lmaWNhdGlvbiBhY2N1cmFjeSwgdGhlcmUgYXJlIG1hbnkgb3RoZXIgbWV0cmljcyB0aGF0IGFyZSB3aWRlbHkgdXNlZCB0byBleGFtaW5lIHRoZSBwZXJmb3JtYW5jZSBvZiBhIGNsYXNzaWZpY2F0aW9uIG1vZGVsLCBpbmNsdWRpbmc6CgoqKlByZWNpc2lvbioqLCB3aGljaCBpcyB0aGUgcHJvcG9ydGlvbiBvZiB0cnVlIHBvc2l0aXZlcyBhbW9uZyBhbGwgdGhlIGluZGl2aWR1YWxzIHRoYXQgaGF2ZSBiZWVuIHByZWRpY3RlZCB0byBiZSBkaWFiZXRlcy1wb3NpdGl2ZSBieSB0aGUgbW9kZWwuIFRoaXMgcmVwcmVzZW50cyB0aGUgYWNjdXJhY3kgb2YgYSBwcmVkaWN0ZWQgcG9zaXRpdmUgb3V0Y29tZS4gUHJlY2lzaW9uID0gVHJ1ZVBvc2l0aXZlcy8oVHJ1ZVBvc2l0aXZlcyArIEZhbHNlUE9zaXRpdmVzKQoKKipTZW5zaXRpdml0eSAob3IgUmVjYWxsKSoqLCB3aGljaCBpcyB0aGUgKipUcnVlIFBvc2l0aXZlIFJhdGUqKiAoVFBSKSBvciB0aGUgcHJvcG9ydGlvbiBvZiBpZGVudGlmaWVkIHBvc2l0aXZlcyBhbW9uZyB0aGUgZGlhYmV0ZXMtcG9zaXRpdmUgcG9wdWxhdGlvbiAoY2xhc3MgPSAxKS4gU2Vuc2l0aXZpdHkgPSBUcnVlUE9zaXRpdmVzLyhUcnVlUG9zaXRpdmVzICsgRmFsc2VOZWdhdGl2ZXMpLgoKKipTcGVjaWZpY2l0eSoqLCB3aGljaCBtZWFzdXJlcyB0aGUgKipUcnVlIE5lZ2F0aXZlIFJhdGUqKiAoVE5SKSwgdGhhdCBpcyB0aGUgcHJvcG9ydGlvbiBvZiBpZGVudGlmaWVkIG5lZ2F0aXZlcyBhbW9uZyB0aGUgZGlhYmV0ZXMtbmVnYXRpdmUgcG9wdWxhdGlvbiAoY2xhc3MgPSAwKS4gU3BlY2lmaWNpdHkgPSBUcnVlTmVnYXRpdmVzLyhUcnVlTmVnYXRpdmVzICsgRmFsc2VOZWdhdGl2ZXMpLgoKKipGYWxzZSBQT3NpdGl2ZSBSYXRlKiogKEZQUiksIHdoaWNoIHJlcHJlc2VudHMgdGhlIHByb3BvcnRpb24gb2YgaWRlbnRpZmllZCBwb3NpdGl2ZXMgYW1vbmcgdGhlIGhlYWx0aHkgaW5kaXZpZHVhbHMgKGkuZS4gZGlhYmV0ZXMtbmVnYXRpdmUpLiBUaGlzIGNhbiBiZSBzZWVuIGFzIGEgZmFsc2UgYWxhcm0uIFRoZSBGUFIgY2FuIGJlIGFsc28gY2FsY3VsYXRlZCBhcyAxLXNwZWNpZmljaXR5LiBXaGVuIHBvc2l0aXZlcyBhcmUgcmFyZSwgdGhlIEZQUiBjYW4gYmUgaGlnaCwgbGVhZGluZyB0byB0aGUgc2l0dWF0aW9uIHdoZXJlIGEgcHJlZGljdGVkIHBvc2l0aXZlIGlzIG1vc3QgbGlrZWx5IGEgbmVnYXRpdmUuCgpgYGB7cn0KY29uZnVzaW9uTWF0cml4KG9ic2VydmVkLmNsYXNzZXMyLCBwcmVkaWN0ZWQuY2xhc3Nlc3MyLCBwb3NpdGl2ZSA9ICJwb3MiKQpgYGAKCgo+IEluIG91ciBleGFtcGxlLCB0aGUgc2Vuc2l0aXZpdHkgaXMgfjU4JSwgdGhhdCBpcyB0aGUgcHJvcG9ydGlvbiBvZiBkaWFiZXRlcy1wb3NpdGl2ZSBpbmRpdmlkdWFscyB0aGF0IHdlcmUgY29ycmVjdGx5IGlkZW50aWZpZWQgYnkgdGhlIG1vZGVsIGFzIGRpYWJldGVzLXBvc2l0aXZlLgpUaGUgc3BlY2lmaWNpdHkgb2YgdGhlIG1vZGVsIGlzIH45MiUsIHRoYXQgaXMgdGhlIHByb3BvcnRpb24gb2YgZGlhYmV0ZXMtbmVnYXRpdmUgaW5kaXZpZHVhbHMgdGhhdCB3ZXJlIGNvcnJlY3RseSBpZGVudGlmaWVkIGJ5IHRoZSBtb2RlbCBhcyBkaWFiZXRlcy1uZWdhdGl2ZS4KVGhlIG1vZGVsIHByZWNpc2lvbiBvciB0aGUgcHJvcG9ydGlvbiBvZiBwb3NpdGl2ZSBwcmVkaWN0ZWQgdmFsdWUgaXMgNzklLgoKCiMjIyBST0MgY3VydmUKClRoZSAqKlJPQyBjdXJ2ZSAob3IgcmVjZWl2ZXIgb3BlcmF0aW5nIGNoYXJhY3RlcmlzdGljcyBjdXJ2ZSkqKiBpcyBhcG9wdWxhciBncmFwaGljYWwgbWVhc3VyZSBmb3IgYXNzZXNzaW5nIHRoZSBwZXJmb3JtYW5jZSBvciB0aGUgYWNjdXJhY3kgb2YgYSBjbGFzc2lmaWVyLCB3aGljaCBjb3JyZXNwb25kcyB0byB0aGUgdG90YWwKCiMjIyBDb21wdXRpbmcgYW5kIHBsb3R0aW5nIFJPQyBjdXJ2ZQoKVGhlIFJPQyBhbmFseXNpcyBjYW4gYmUgZWFzaWx5IHBlcmZvcm1lZCB1c2luZyB0aGUgUiBwYWNrYWdlIHBST0MKCmBgYHtyfQpsaWJyYXJ5KHBST0MpCnJlcy5yb2MgPC0gcm9jKG9ic2VydmVkLmNsYXNzZXMyLHByZWljdGlvbnMucHJvYmFiaWxpdGllczIpCnBsb3Qucm9jKHJlcy5yb2MsIHByaW50LmF1YyA9IFRSVUUpCmBgYAoKYGBge3J9CiMgRXh0cmFjdCBzb21lIGludGVyZXN0aW5nIHJlc3VsdHMKcm9jLmRhdGEgPC0gZGF0YV9mcmFtZSgKICB0aHJlc2hvbGRzID0gcmVzLnJvYyR0aHJlc2hvbGRzLAogIHNlbnNpdGl2aXR5ID0gcmVzLnJvYyRzZW5zaXRpdml0aWVzLAogIHNwZWNpZmljaXR5ID0gcmVzLnJvYyRzcGVjaWZpY2l0aWVzCikKCiMgR2V0IHRoZSBwcm9iYWJpbGl0eSB0aHJlc2hvbGQgZm9yIHNwZWNpZmljaXR5ID0gMC42Cgpyb2MuZGF0YSAlPiUgZmlsdGVyKHNwZWNpZmljaXR5ID49IDAuNikKYGBgCgoKCgpgYGB7cn0KcGxvdC5yb2MocmVzLnJvYywgcHJpbnQuYXVjID0gVFJVRSwgcHJpbnQudGhyZXMgPSAiYmVzdCIpCmBgYAoKYGBge3J9CnBsb3Qucm9jKHJlcy5yb2MsIHByaW50LnRocmVzID0gYygwLjMsMC41LDAuNykpCmBgYAoKCgoK